Merge branch 'release-0.9.0'

LongTailBio · May 11, 2018 · 8c2dfdb · 8c2dfdb
2 parents 4525e42 + ca609b4
commit 8c2dfdb
Show file tree

Hide file tree

Showing 346 changed files with 10,553 additions and 1,400 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,7 +1,81 @@
 version: 2
 
+# Common steps for building server Docker images
+build_steps: &build_steps
+  docker:
+    - image: circleci/node:9.2.0
+
+  steps:
+    - checkout
+
+    - setup_remote_docker
+
+    - run:
+        name: Set COMMIT env var
+        command: echo 'export COMMIT=${CIRCLE_SHA1::8}' >> $BASH_ENV
+
+    - run:
+        name: Sign in to Docker Hub
+        command: docker login -u $DOCKER_ID -p $DOCKER_PASSWORD
+
+    - run:
+        name: Build and push mongo-db
+        environment:
+          DB_SERVICE: mongo
+        command: |
+          docker build ./database_docker/mongo_db -t $DB_SERVICE:$COMMIT
+          docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
+          docker push $DOCKER_ORG/$DB_SERVICE
+
+    - run:
+        name: Build and push metagenscope-db
+        environment:
+          DB_SERVICE: metagenscope-db
+        command: |
+          docker build ./database_docker/postgres_db -t $DB_SERVICE:$COMMIT
+          docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
+          docker push $DOCKER_ORG/$DB_SERVICE
+
+    - run:
+        name: Build and push redis
+        environment:
+          DB_SERVICE: redis
+        command: |
+          docker build ./database_docker/redis -t $DB_SERVICE:$COMMIT
+          docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
+          docker push $DOCKER_ORG/$DB_SERVICE
+
+    - run:
+        name: Build and push rabbitmq
+        environment:
+          DB_SERVICE: rabbitmq
+        command: |
+          docker build ./database_docker/rabbitmq -t $DB_SERVICE:$COMMIT
+          docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
+          docker push $DOCKER_ORG/$DB_SERVICE
+
+    - run:
+        name: Build and push metagenscope-worker
+        environment:
+          DB_SERVICE: metagenscope-worker
+        command: |
+          docker build . -f Dockerfile-worker -t $DB_SERVICE:$COMMIT
+          docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
+          docker push $DOCKER_ORG/$DB_SERVICE
+
+    - run:
+        name: Build and push metagenscope-service
+        environment:
+          MAIN_SERVICE: metagenscope-service
+        command: |
+          docker build . -t $MAIN_SERVICE:$COMMIT
+          docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG
+          docker push $DOCKER_ORG/$MAIN_SERVICE
+
+
+# CircleCI Jobs
 jobs:
-  run-tests:
+  test_app:
     docker:
       - image: circleci/python:3.6.3-jessie
         environment:
@@ -37,6 +111,17 @@ jobs:
             . venv/bin/activate
             pip install -r requirements.txt
 
+      - save_cache:
+          paths:
+            - ./venv
+          key: v1-dependencies-{{ checksum "requirements.txt" }}
+
+      - run:
+          name: Lint app
+          command: |
+            . venv/bin/activate
+            make lint
+
       - run:
           name: Wait for DB
           command: dockerize -wait tcp://localhost:5432 -timeout 1m
@@ -47,14 +132,8 @@ jobs:
             . venv/bin/activate
             python manage.py recreate_db
 
-      - save_cache:
-          paths:
-            - ./venv
-          key: v1-dependencies-{{ checksum "requirements.txt" }}
-
-      # run tests!
       - run:
-          name: run tests
+          name: Run application tests
           command: |
             . venv/bin/activate
             python manage.py cov
@@ -63,56 +142,13 @@ jobs:
           path: htmlcov
           destination: test-reports
 
-  build_emptyfish:
-    docker:
-      - image: circleci/node:9.2.0
-
+  build_staging:
+    <<: *build_steps
     environment:
-      TAG: emptyfish
+      TAG: staging
       DOCKER_ORG: metagenscope
-      REACT_APP_METAGENSCOPE_SERVICE_URL: https://www.emptyfish.net
-
-    steps:
-      - checkout
-
-      - setup_remote_docker
-
-      - run:
-          name: Set COMMIT env var
-          command: echo 'export COMMIT=${CIRCLE_SHA1::8}' >> $BASH_ENV
-
-      - run:
-          name: Sign in to Docker Hub
-          command: docker login -u $DOCKER_ID -p $DOCKER_PASSWORD
-
-      - run:
-          name: Build and push mongo-db
-          environment:
-            DB_SERVICE: mongo
-          command: |
-            docker build ./database_docker/mongo_db -t $DB_SERVICE:$COMMIT
-            docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
-            docker push $DOCKER_ORG/$DB_SERVICE
-
-      - run:
-          name: Build and push metagenscope-db
-          environment:
-            DB_SERVICE: metagenscope-db
-          command: |
-            docker build ./database_docker/postgres_db -t $DB_SERVICE:$COMMIT
-            docker tag $DB_SERVICE:$COMMIT $DOCKER_ORG/$DB_SERVICE:$TAG
-            docker push $DOCKER_ORG/$DB_SERVICE
-
-      - run:
-          name: Build and push metagenscope-service
-          environment:
-            MAIN_SERVICE: metagenscope-service
-          command: |
-            docker build . -t $MAIN_SERVICE:$COMMIT
-            docker tag $MAIN_SERVICE:$COMMIT $DOCKER_ORG/$MAIN_SERVICE:$TAG
-            docker push $DOCKER_ORG/$MAIN_SERVICE
 
-  deploy_emptyfish:
+  deploy_staging:
     docker:
       - image: circleci/node:9.2.0
 
@@ -123,20 +159,42 @@ jobs:
           name: Deploy to emptyfish.net
           command: |
             set -x
-            echo "$DROPLET_IP $DROPLET_HOST_KEY" > ~/tmp_auth_hosts
-            ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $DROPLET_USER@$DROPLET_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh"
+            echo "$STAGING_MACHINE_IP $STAGING_MACHINE_HOST_KEY" > ~/tmp_auth_hosts
+            ssh -A -o "UserKnownHostsFile ~/tmp_auth_hosts" $STAGING_MACHINE_USER@$STAGING_MACHINE_IP "cd /home/metagenscope/metagenscope-app && sh deploy.sh"
+
+  build_master:
+    <<: *build_steps
+    environment:
+      TAG: latest
+      DOCKER_ORG: metagenscope
+
 
 workflows:
   version: 2
-  test-and-deploy:
+
+  app_cd:
     jobs:
-      - run-tests:
-          context: org-global
-      - build_emptyfish:
-          context: org-global
+      - test_app:
+          context: metagenscope-staging
+      - build_staging:
+          context: metagenscope-staging
+          filters:
+            branches:
+              only: develop
+          requires:
+            - test_app
+      - deploy_staging:
+          context: metagenscope-staging
+          filters:
+            branches:
+              only: develop
           requires:
-            - run-tests
-      - deploy_emptyfish:
-          context: org-global
+            - build_app_staging
+            - build_worker_staging
+      - build_master:
+          context: metagenscope-staging
+          filters:
+            branches:
+              only: master
           requires:
-            - build_emptyfish
+            - test_app
diff --git a/.pylintrc b/.pylintrc
@@ -50,7 +50,7 @@ confidence=
 # --enable=similarities". If you want to run only the classes checker, but have
 # no Warning level messages displayed, use"--disable=all --enable=classes
 # --disable=W"
-disable=print-statement,parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call
+disable=parameter-unpacking,unpacking-in-except,old-raise-syntax,backtick,long-suffix,old-ne-operator,old-octal-literal,import-star-module-level,raw-checker-failed,bad-inline-option,locally-disabled,locally-enabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,apply-builtin,basestring-builtin,buffer-builtin,cmp-builtin,coerce-builtin,execfile-builtin,file-builtin,long-builtin,raw_input-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,no-absolute-import,old-division,dict-iter-method,dict-view-method,next-method-called,metaclass-assignment,indexing-exception,raising-string,reload-builtin,oct-method,hex-method,nonzero-method,cmp-method,input-builtin,round-builtin,intern-builtin,unichr-builtin,map-builtin-not-iterating,zip-builtin-not-iterating,range-builtin-not-iterating,filter-builtin-not-iterating,using-cmp-argument,eq-without-hash,div-method,idiv-method,rdiv-method,exception-message-attribute,invalid-str-codec,sys-max-int,bad-python3-import,deprecated-string-function,deprecated-str-translate-call
 
 # Enable the message, report, category or checker with the given id(s). You can
 # either give multiple identifier separated by comma (,) or put this option
@@ -230,7 +230,7 @@ notes=FIXME,XXX,TODO
 [SIMILARITIES]
 
 # Ignore comments when computing similarities.
-ignore-comments=yes
+ignore-comments=no
 
 # Ignore docstrings when computing similarities.
 ignore-docstrings=yes
@@ -239,7 +239,7 @@ ignore-docstrings=yes
 ignore-imports=no
 
 # Minimum lines number of a similarity.
-min-similarity-lines=4
+min-similarity-lines=6
 
 
 [SPELLING]

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+
+## 0.9.0 - 2018-05-11
 ### Added
 - Basic Docker configuration.
 - PostgreSQL Docker and SQLAlchemy configuration.
@@ -24,4 +26,5 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Added
 - Basic Flask project structure.
 
-[Unreleased]: https://github.com/bchrobot/metagenscope-server/compare/v0.0.1...HEAD
+[Unreleased]: https://github.com/LongTailBio/metagenscope-server/compare/v0.9.0...HEAD
+[0.9.0]: https://github.com/LongTailBio/metagenscope-server/compare/v0.0.1...v0.9.0
diff --git a/Dockerfile b/Dockerfile
@@ -5,13 +5,16 @@ RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
 
 # Add requirements (to leverage Docker cache)
-ADD ./requirements.txt /usr/src/app/requirements.txt
+COPY ./requirements.txt /usr/src/app/requirements.txt
 
 # Install requirements
 RUN pip install -r requirements.txt
 
 # Add app
-ADD . /usr/src/app
+COPY . /usr/src/app
+
+# Make startup scripts executable
+RUN chmod +x /usr/src/app/startup.sh /usr/src/app/wait-for-it.sh
 
 # Run server
 CMD python manage.py runserver -h 0.0.0.0
diff --git a/Dockerfile-local b/Dockerfile-local
diff --git a/Dockerfile-worker b/Dockerfile-worker
@@ -0,0 +1,24 @@
+FROM python:3.6.1
+
+# Set working directory
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# Add requirements (to leverage Docker cache)
+COPY ./requirements.txt /usr/src/app/requirements.txt
+
+# Install requirements
+RUN pip install -r requirements.txt
+
+# Copy source code
+COPY . /usr/src/app
+
+# Make startup scripts executable
+RUN chmod +x /usr/src/app/startup.sh /usr/src/app/wait-for-it.sh
+
+# Switch to celery user
+RUN useradd -ms /bin/bash celery; chown -R celery /usr/src/app
+USER celery
+
+# Run the worker
+CMD celery worker -A worker.celery --loglevel=info
diff --git a/Makefile b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: clean-pyc clean-build clean lint lint-tests lint-seed test cov
+.PHONY: clean-pyc clean-build clean lint-app lint-tests lint-seed lint-worker lint test cov
 .DEFAULT_GOAL: help
 
 help:
@@ -26,7 +26,7 @@ clean-pyc:
 	find . -name '*~' -exec rm -f {} +
 	find . -name '__pycache__' -exec rm -fr {} +
 
-lint:
+lint-app:
 	pylint --rcfile=.pylintrc app -f parseable -r n && \
 	pycodestyle app --max-line-length=120 && \
 	pydocstyle app
@@ -41,6 +41,16 @@ lint-seed:
 	pycodestyle seed --max-line-length=120 && \
 	pydocstyle seed
 
+lint-worker:
+	pylint --rcfile=.pylintrc worker -f parseable -r n && \
+	pycodestyle worker --max-line-length=120 && \
+	pydocstyle worker
+
+lint:
+	pylint --rcfile=.pylintrc app tests seed worker -f parseable -r n && \
+	pycodestyle app tests seed worker --max-line-length=120 && \
+	pydocstyle app tests seed worker
+
 test: lint
 	python manage.py test