diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 38235556..a21e07d8 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,13 +4,11 @@ stages: - test style - validate format - test stories + - build requirements + - build coach - build - deploy -variables: - BOT_LATEST_IMAGE: $CI_REGISTRY_IMAGE/bot:latest - WEB_LATEST_IMAGE: $CI_REGISTRY_IMAGE/web:latest - .set_ssh_config: &set_ssh_config before_script: - apt-get update -y @@ -26,45 +24,83 @@ test style: run dataset validator: stage: validate format - image: registry.gitlab.com/lappis-unb/services/tais/bot + image: lappis/coach:latest script: - cd coach/ - make run-validator test stories: stage: test stories - image: registry.gitlab.com/lappis-unb/services/tais/bot + image: lappis/bot:latest script: - cd bot/ - - make train > /dev/null 2>&1 - make test-stories + +build requirements: + stage: build requirements + image: docker + tags: + - docker + services: + - docker:dind + script: + - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD + - cd ./docker/bot + - ./build-base.sh publish only: - - merge_requests + refs: + - master + changes: + - ./docker/bot/requirements.txt + environment: homolog + +build coach: + stage: build coach + image: docker + tags: + - docker + services: + - docker:dind + script: + - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD + - docker build . -f docker/bot/coach.Dockerfile -t lappis/coach:latest + - docker push lappis/coach:latest + only: + refs: + - master + changes: + - ./coach/data + environment: homolog build bot: stage: build image: docker + tags: + - docker services: - docker:dind script: - - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY - - docker build -f docker/bot/bot.Dockerfile -t $BOT_LATEST_IMAGE . - - docker push $BOT_LATEST_IMAGE + - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD + - docker build -f docker/bot/bot.Dockerfile -t lappis/bot:latest . + - docker push lappis/bot:latest only: - master + environment: homolog build web: stage: build image: docker + tags: + - docker services: - docker:dind script: - - docker login -u "gitlab-ci-token" -p "$CI_JOB_TOKEN" $CI_REGISTRY - - - docker build -f docker/web/web.Dockerfile -t $WEB_LATEST_IMAGE . - - docker push $WEB_LATEST_IMAGE + - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD + - docker build -f docker/web/web.Dockerfile -t lappis/bot-web:latest . + - docker push lappis/bot-web:latest only: - master + environment: homolog deploy bot to homolog: stage: deploy diff --git a/Makefile b/Makefile index b6b26806..850daf1e 100644 --- a/Makefile +++ b/Makefile @@ -1,2 +1,6 @@ train: docker build . -f docker/bot/coach.Dockerfile -t lappis/coach:latest + docker-compose build bot + +test-dialogue: + docker-compose run --rm bot make e2e \ No newline at end of file diff --git a/bot/Makefile b/bot/Makefile index 4b9cd85d..84dcc807 100644 --- a/bot/Makefile +++ b/bot/Makefile @@ -1,15 +1,16 @@ run-rocketchat: + make config-bot python3 run-rocketchat.py run-api: - python3 -m rasa_core.run -d models/dialogue -u models/nlu/current --debug \ + python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug \ --port 5005 --enable_api run-console: - python3 -m rasa_core.run -d models/dialogue -u models/nlu/current --debug + python3 -m rasa_core.run -d /models/dialogue -u /models/nlu/current --debug evaluate: - python3 -m rasa_core.evaluate -d models/dialogue -s data/core/ + python3 -m rasa_core.evaluate -d /models/dialogue -s data/core/ run-validator: python3 validator.py --intents data/intents/ --stories data/stories --domain domain.yml @@ -17,11 +18,14 @@ run-validator: # Force Makefile to execute e2e command every time .PHONY: all e2e clean e2e: - python3 -m rasa_core.evaluate default --core models/dialogue --nlu models/nlu/current --stories e2e/e2e_stories_1.md --e2e - python3 -m rasa_core.evaluate default --core models/dialogue --nlu models/nlu/current --stories e2e/e2e_stories_2.md --e2e + python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories_1.md --e2e + python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories_2.md --e2e e2e-test: - python3 -m rasa_core.evaluate default --core models/dialogue --nlu models/nlu/current --stories e2e/e2e_stories_erro.md --e2e + python3 -m rasa_core.evaluate default --core /models/dialogue --nlu /models/nlu/current --stories e2e/e2e_stories_erro.md --e2e test-stories: python3 test_stories.py --stories e2e/ --e2e + +config-bot: + python /scripts/bot_config.py -r ${ROCKETCHAT_URL} -an ${ROCKETCHAT_ADMIN_USERNAME} -ap ${ROCKETCHAT_ADMIN_PASSWORD} -bu ${ROCKETCHAT_BOT_USERNAME} -bp ${ROCKETCHAT_BOT_PASSWORD} diff --git a/bot/run-rocketchat.py b/bot/run-rocketchat.py index 18974ab2..58f6ea90 100644 --- a/bot/run-rocketchat.py +++ b/bot/run-rocketchat.py @@ -56,4 +56,4 @@ def run(core_dir, nlu_dir): if __name__ == '__main__': - run('models/dialogue', 'models/nlu/current') + run('/models/dialogue', '/models/nlu/current') diff --git a/bot/test_stories.py b/bot/test_stories.py index 7b0ef2d7..222ce10d 100644 --- a/bot/test_stories.py +++ b/bot/test_stories.py @@ -21,8 +21,8 @@ FAILED_COLOR = utils.bcolors.FAIL BLUE_COLOR = utils.bcolors.OKBLUE BOLD_COLOR = utils.bcolors.BOLD -CORE_DIR = 'models/dialogue' -NLU_DIR = 'models/nlu/current' +CORE_DIR = '/models/dialogue' +NLU_DIR = '/models/nlu/current' parser = argparse.ArgumentParser() diff --git a/coach/Makefile b/coach/Makefile index 0558043c..c5298204 100644 --- a/coach/Makefile +++ b/coach/Makefile @@ -9,7 +9,7 @@ train: train-nlu train-core train-online: python3 -m rasa_core.train -d domain.yml -s data/stories -o \ - models/dialogue --epochs 15 --online --nlu models/nlu/current/ + /src_models/dialogue --epochs 15 --online --nlu /src_models/nlu/current/ run-validator: python3 validator.py --intents data/intents/ --stories data/stories --domain domain.yml diff --git a/coach/domain.yml b/coach/domain.yml index 76706ba9..6be3585f 100644 --- a/coach/domain.yml +++ b/coach/domain.yml @@ -111,7 +111,7 @@ templates: - text: A minha missão é te guiar para a inscrição da sua proposta, Se você já sabe isso é porque você já tem um conhecimento avançado e talvez eu não consiga sanar as suas dúvidas. - Mas podemos continuar conversando sobre alguns conceitos básicos da lei Rouanet =) + Mas podemos continuar conversando sobre alguns conceitos básicos da Lei de Incentivo à Cultura =) utter_expressoes_indesejadas: - text: | diff --git a/coach/policy_config.yml b/coach/policy_config.yml index ba85c4ce..c0168c42 100644 --- a/coach/policy_config.yml +++ b/coach/policy_config.yml @@ -1,6 +1,6 @@ policies: - name: KerasPolicy - epochs: 10 + epochs: 7 batch_size: 10 featurizer: - name: FullDialogueTrackerFeaturizer diff --git a/docker-compose.yml b/docker-compose.yml index 0120a638..742e22a8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,28 +32,23 @@ services: context: . dockerfile: ./docker/bot/bot.Dockerfile restart: unless-stopped + command: make run-rocketchat + # OR + # command: make run-console + depends_on: + - rocketchat env_file: - ./docker/bot/bot.env ports: - 5005:5005 volumes: - - models:/bot/models/:ro - ./bot:/bot - ./scripts:/scripts - depends_on: - - coach - - coach: - image: lappis/coach:latest - command: cp -a /src_models/. /models/ - volumes: - - models:/models/ - - ./coach:/coach - - ./scripts:/scripts + # =================================== Web =================================== web: - image: registry.gitlab.com/lappis-unb/services/tais/web + image: lappis/web:latest build: context: . dockerfile: ./docker/web/web.Dockerfile @@ -192,5 +187,3 @@ volumes: esdata: driver: local esbackup: - models: - diff --git a/docker/bot/bot.Dockerfile b/docker/bot/bot.Dockerfile index a3bd8691..23ab2d12 100644 --- a/docker/bot/bot.Dockerfile +++ b/docker/bot/bot.Dockerfile @@ -1,6 +1,9 @@ +FROM lappis/coach:latest as coach + FROM lappis/botrequirements:latest COPY ./bot /bot +COPY --from=coach /src_models/ /models/ COPY ./scripts /scripts WORKDIR /bot @@ -19,8 +22,3 @@ ENV ROCKETCHAT_URL=rocketchat:3000 \ ELASTICSEARCH_URL=elasticsearch:9200 RUN find . | grep -E "(__pycache__|\.pyc|\.pyo$)" | xargs rm -rf - -CMD python /scripts/bot_config.py -r $ROCKETCHAT_URL \ - -an $ROCKETCHAT_ADMIN_USERNAME -ap $ROCKETCHAT_ADMIN_PASSWORD \ - -bu $ROCKETCHAT_BOT_USERNAME -bp $ROCKETCHAT_BOT_PASSWORD \ - && make run-rocketchat diff --git a/scripts/deploy_bot.sh b/scripts/deploy_bot.sh index 4fe0de37..6d532a91 100755 --- a/scripts/deploy_bot.sh +++ b/scripts/deploy_bot.sh @@ -1,9 +1,9 @@ #!/bin/bash sshpass -p $1 ssh -o StrictHostKeyChecking=no root@$2 <<-'ENDSSH' - cd rouana/ - docker-compose -f ./beta.docker-compose.yml stop bot - docker-compose -f ./beta.docker-compose.yml rm -f bot - docker rmi -f registry.gitlab.com/lappis-unb/services/tais/bot - docker-compose -f ./beta.docker-compose.yml up -d bot + cd rouana/ + docker-compose stop bot + docker-compose rm -f bot + docker-compose pull bot + docker-compose up -d bot ENDSSH diff --git a/scripts/deploy_web.sh b/scripts/deploy_web.sh index 529d5649..76cc527f 100755 --- a/scripts/deploy_web.sh +++ b/scripts/deploy_web.sh @@ -2,8 +2,8 @@ sshpass -p $1 ssh -o StrictHostKeyChecking=no root@$2 <<-'ENDSSH' cd rouana/ - docker-compose -f ./beta.docker-compose.yml stop web - docker-compose -f ./beta.docker-compose.yml rm -f web - docker rmi -f registry.gitlab.com/lappis-unb/services/tais/web - docker-compose -f ./beta.docker-compose.yml up -d web + docker-compose stop web + docker-compose rm -f web + docker-compose pull web + docker-compose up -d web ENDSSH