Merge branch 'develop_v4' of github.com:inasafe/inasafe-django into d…

…evelop

.gitignore

@@ -108,7 +108,10 @@ deployment/docker-compose.override.yml
 !deployment/sftp_backup/backups/pg/README.md
 django_project/realtime/local_settings.py
 django_project/celerybeat.pid
+django_project/celerybeat-schedule
 django_project/temp
+deployment/realtime-hazard
+!deployment/realtime-hazard/shakemaps/.gitkeep
 
 sftp_media_credential.env
 sftp_pg_credential.env
@@ -130,3 +133,11 @@ django_project/media_test
 # pickled indicator
 django_project/.run/
 .realtime_broker_indicator.pickle
+deployment/backups/
+deployment/headless-data/contexts/
+deployment/headless-data/inasafe/
+deployment/headless-data/outputs/
+deployment/headless-data/qgis-templates/
+deployment/headless-data/test-hazards/
+deployment/reports/
+.coverage*

.travis.yml

@@ -1,43 +1,73 @@
+sudo: required
+
 language: python
 
 services:
-  - postgresql
+  - docker
 
 env:
   global:
-    - DATABASE_URL='postgres://postgres:@localhost:5432/test_db'
-    - SECRET_KEY='tT\xd7\xb06\xf7\x9b\xff\x0fZL\xca\xca\x11\xefM\xacr\xfb\xdf\xca\x9b'
-    - DJANGO_SETTINGS_MODULE=core.settings.test_travis
+    - ON_TRAVIS=True
+  matrix:
+    - INTEGRATION_TEST=False
+    - INTEGRATION_TEST=True
+
+virtualenv:
+  system_site_packages: true
 
 python:
   - '2.7'
 
-addons:
-  postgresql: "9.3"
-  apt:
-    packages:
-      - postgresql-9.3-postgis-2.3
-
 install:
-  - pip install coveralls
-  - pip install -r REQUIREMENTS-dev.txt
-
-
-before_script:
-  - psql -c 'create database test_db;' -U postgres
-  - psql -c 'CREATE EXTENSION postgis;' -U postgres -d test_db
-  - cd django_project
+  - sudo apt install zip wget
+  - pip install -r REQUIREMENTS-TRAVIS.txt
+  - pushd deployment
+  - cp ansible/development/group_vars/all.travis.yml ansible/development/group_vars/all.yml
+  - make setup-ansible ANSIBLE_ARGS="--extra-vars='ansible_sudo_pass=\"\"' --extra-vars='pycharm_version=\"\"' --skip-tags=development"
+  - make build
+  - make pull-external-services
+  - make up
+  - make status
+  - mkdir -p headless-data/qgis-templates
+  - mkdir -p headless-data/contexts
+  # Retrieve latest templates
+  - sudo git clone https://github.com/inasafe/inasafe-realtime-resources.git headless-data/qgis-templates
+  - cat docker-compose.override.yml
+  - until make migrate; do echo "Retrying"; sleep 5; done
+  - popd
 
 script:
-  - flake8
-  # Split this, because somehow test failed when using:
-  # coverage run manage.py test
-  - coverage run manage.py test realtime
-  - coverage run manage.py test user_map
+  - pushd deployment
+  - make check-flake8
+  - if [ "$INTEGRATION_TEST" = "False" ]; then make coverage-tests; fi
+  # Do integration tests
+  - make inasafe-worker-stop
+  - make realtime-worker
+  - make headless-worker
+  - sleep 30
+  # Retrieve analysis context data
+  - if [ "$INTEGRATION_TEST" = "True" ]; then make retrieve-context-data; fi
+  - if [ "$INTEGRATION_TEST" = "True" ]; then make integration-tests; fi
+  - popd
+
+after_script:
+  # Clean up
+  - pushd deployment
+  - make down
+  - popd
 
 after_success:
-  - coveralls
+  - pushd django_project
+  - coverage combine
+  # Fix path (codecov.yml doesn't work)
+  # Replace parent directory to travis directory, because we are running
+  # coverage using docker mount
+  - sed -i 's/home\/web\/django_project/home\/travis\/build\/inasafe\/inasafe-django\/django_project/g' .coverage
+  - cat .coverage
+  - codecov
+  - cat coverage.xml
+  - popd
 
 email:
- - tim@kartoza.com
  - lana.pcfre@gmail.com
+ - imajimatika@gmail.com

README.md

@@ -18,7 +18,9 @@ These badges reflect the current status of our development branch:
 
 Tests status: [![Build Status](https://travis-ci.org/inasafe/inasafe-django.svg)](https://travis-ci.org/inasafe/inasafe-django)
 
-Coverage status: [![Coverage Status](https://coveralls.io/repos/inasafe/inasafe-django/badge.png?branch=develop)](https://coveralls.io/r/inasafe/inasafe-django?branch=develop)
+Coverage status: [![codecov](https://codecov.io/gh/inasafe/inasafe-django/branch/develop_v4/graph/badge.svg)](https://codecov.io/gh/inasafe/inasafe-django)
+
+
 
 Development status: [![Stories in Ready](https://badge.waffle.io/AIFDR/inasafe-django.svg?label=ready&title=Ready)](http://waffle.io/inasafe/inasafe-django) [![Stories in Ready](https://badge.waffle.io/AIFDR/inasafe-django.svg?label=In%20Progress&title=In%20Progress)](http://waffle.io/inasafe/inasafe-django)
 
@@ -161,6 +163,9 @@ Just adapt pgAdmin 3 connection using this information.
 
 
 ### Run Django using Pycharm
+
+For more information on how to use PyCharm features on this project refer to
+[PyCharm Readme](deployment/README-PyCharm.md)
 
 After creating remote docker interpreter for this project, you can use the 
 interpreter to create django configuration.

REQUIREMENTS-TRAVIS.txt

@@ -0,0 +1,3 @@
+ansible
+codecov
+flake8

deployment/Makefile

@@ -19,7 +19,7 @@ down:
 	@echo "------------------------------------------------------------------"
 	@docker-compose -p $(PROJECT_ID) down
 
-complete-deploy: up wait migrate collectstatic makemessages compilemessages inasafe-worker indicator-worker reload status
+complete-deploy: up wait migrate collectstatic makemessages compilemessages inasafe-worker indicator-worker realtime-worker headless-worker reload status
 
 wait:
 	@echo
@@ -46,6 +46,86 @@ setup-ansible:
 	@echo "Setup configurations using ansible"
 	@ansible-playbook -i ansible/development/hosts ansible/development/site.yml -v $(ANSIBLE_ARGS)
 
+check-flake8:
+	@echo "Check flake 8"
+	@flake8 ../django_project
+
+pull-external-services:
+	@echo "Pull InaSAFE Realtime"
+	@docker-compose -p $(PROJECT_ID) pull realtime-worker
+	@echo "Pull InaSAFE Headless"
+	@docker-compose -p $(PROJECT_ID) pull headless-worker
+
+pull-external-services-docker:
+	@echo "Pull InaSAFE Realtime"
+	@docker pull inasafe/realtime_hazard-processor
+	@echo "Pull InaSAFE Headless"
+	@docker pull inasafe/headless_processor
+
+coverage-tests:
+	@docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test realtime --noinput --liveserver=0.0.0.0:8080 -v 2
+
+integration-tests-worker:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running integration tests celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d integration-test-worker
+
+integration-tests-worker-restart:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Restart integration tests celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) restart integration-test-worker
+
+integration-tests-worker-stop:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Stop integration tests celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) stop integration-test-worker
+
+integration-tests-worker-logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running integration tests celery worker logs"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 integration-test-worker
+
+combined-worker-logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing combined worker logs"
+	@echo "------------------------------------------------------------------"
+	@{ make realtime-worker-logs & make headless-worker-logs & make inasafe-worker-logs; }
+
+combined-integration-worker-logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Showing combined integration worker logs"
+	@echo "------------------------------------------------------------------"
+	@{ make realtime-worker-logs & make headless-worker-logs & make integration-tests-worker-logs & make integration-tests-runner-logs; }
+
+integration-tests: realtime-worker headless-worker integration-tests-worker
+	@docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test --liveserver=0.0.0.0:8080 --noinput realtime.tests.test_scenarios.TestEarthquakeTasks
+	@make integration-tests-worker-restart
+	@docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test --liveserver=0.0.0.0:8080 --noinput realtime.tests.test_scenarios.TestFloodTasks
+	@make integration-tests-worker-restart
+	@docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test --liveserver=0.0.0.0:8080 --noinput realtime.tests.test_scenarios.TestAshTasks
+
+integration-tests-with-logs: realtime-worker headless-worker integration-tests-worker
+	@-{ make headless-worker-logs & make realtime-worker-logs & make integration-tests-worker-logs & docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test --liveserver=0.0.0.0:8080 --noinput realtime.tests.test_scenarios.TestEarthquakeTasks; }
+	@-make integration-tests-worker-restart
+	@-{ make headless-worker-logs & make realtime-worker-logs & make integration-tests-worker-logs & docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test --liveserver=0.0.0.0:8080 --noinput realtime.tests.test_scenarios.TestFloodTasks; }
+	@-make integration-tests-worker-restart
+	@-{ make headless-worker-logs & make realtime-worker-logs & make integration-tests-worker-logs & docker-compose -p $(PROJECT_ID) exec uwsgi coverage run -p --branch --source='.' manage.py test --liveserver=0.0.0.0:8080 --noinput realtime.tests.test_scenarios.TestAshTasks; }
+
+retrieve-context-data:
+	@wget --no-check-certificate https://inasafe.bnpb.go.id/media/context-data/context-data.zip
+	@mv context-data.zip headless-data/contexts
+	@pushd headless-data/contexts && unzip context-data.zip && popd
+
 build:
 	@echo
 	@echo "------------------------------------------------------------------"
@@ -142,7 +222,7 @@ inasafe-worker-stop:
 inasafe-worker-logs:
 	@echo
 	@echo "------------------------------------------------------------------"
-	@echo "Running celery worker "
+	@echo "Running celery worker logs"
 	@echo "------------------------------------------------------------------"
 	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 inasafe-worker
 
@@ -170,10 +250,78 @@ indicator-worker-stop:
 indicator-worker-logs:
 	@echo
 	@echo "------------------------------------------------------------------"
-	@echo "Running celery worker "
+	@echo "Running celery worker logs"
 	@echo "------------------------------------------------------------------"
 	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 indicator-worker
 
+realtime-worker:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running InaSAFE Realtime celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d realtime-worker
+
+realtime-worker-shell:
+	@docker-compose -p $(PROJECT_ID) exec realtime-worker /bin/bash
+
+realtime-worker-restart:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Restart InaSAFE Realtime celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) restart realtime-worker
+
+realtime-worker-stop:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Stop InaSAFE Realtime celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) stop realtime-worker
+
+realtime-worker-logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running InaSAFE Realtime celery worker logs"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 realtime-worker
+
+realtime-shakemap-monitor-logs:
+	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 shakemap-monitor
+
+realtime-shakemap-corrected-monitor-logs:
+	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 shakemap-corrected-monitor
+
+headless-worker:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running InaSAFE Headless celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d headless-worker
+
+headless-worker-shell:
+	@docker-compose -p $(PROJECT_ID) exec headless-worker /bin/bash
+
+headless-worker-restart:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Restart InaSAFE Headless celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) restart headless-worker
+
+headless-worker-stop:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Stop InaSAFE Headless celery worker "
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) stop headless-worker
+
+headless-worker-logs:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running InaSAFE Headless celery worker logs"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) logs -f --tail=50 headless-worker
+
 status:
 	@docker-compose -p $(PROJECT_ID) ps
 
@@ -261,7 +409,7 @@ dbrestore:
 	@echo "Create database gis"
 	@docker-compose -p $(PROJECT_ID) exec db su - postgres -c "createdb -O docker -T template_postgis gis"
 	@echo "Dumping to database gis"
-	@docker-compose -p $(PROJECT_ID) exec dbbackup pg_restore /backups/latest.dmp | docker exec -i $(PROJECT_ID)_db_1 su - postgres -c "psql gis"
+	@docker-compose -p $(PROJECT_ID) exec db "pg_restore /backups/latest.dmp | su - postgres -c 'psql gis'"
 
 dbbackup:
 	@echo
@@ -317,3 +465,32 @@ reload:
 	@echo "Reload django project "
 	@echo "------------------------------------------------------------------"
 	@docker-compose -p $(PROJECT_ID) exec uwsgi uwsgi --reload /tmp/django.pid
+
+contexts-sync:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running contexts btsync"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d contexts-btsync
+
+qgis-templates-sync:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Running QGIS templates btsync"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) up -d qgis-templates-btsync
+
+import-kelurahan-boundary:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Import kelurahan boundary"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py \
+	importfloodboundary /home/headless/contexts/flood/exposure/DKI_Jakarta_Population_Dukcapil_2013_WGS84.shp '' NAMA_KELUR 7 Kelurahan
+
+load-flood-test-data:
+	@echo
+	@echo "------------------------------------------------------------------"
+	@echo "Load flood test data"
+	@echo "------------------------------------------------------------------"
+	@docker-compose -p $(PROJECT_ID) exec uwsgi python manage.py loadfloodtestdata