Merge pull request #62 from joaovitor3/add_ci

Adiciona integração contínua
lappis-unb · Aug 10, 2019 · c27c4dc · c27c4dc
2 parents 8da53c4 + c7ac7c9
commit c27c4dc
Show file tree

Hide file tree

Showing 10 changed files with 192 additions and 80 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -0,0 +1,100 @@
+image: python:3.6-slim
+
+variables:
+  BOT_IMAGE: $DOCKERHUB_USER/bot
+  COACH_IMAGE: $DOCKERHUB_USER/coach
+
+stages:
+  - test style
+  - test stories
+  - build requirements
+  - build coach
+  - build
+
+
+#############################################################
+#################### TEST STYLE #############################
+#############################################################
+test style:
+  stage: test style
+  script:
+    - pip -V
+    - python -V
+    - pip install -r requirements.txt
+    - flake8 --exclude venv
+
+
+#############################################################
+############### TEST STORIES ################################
+#############################################################
+test stories:
+  stage: test stories
+  image: docker
+  tags:
+    - docker
+  services:
+    - docker:dind
+  script:
+    - docker build -f docker/coach.Dockerfile -t $COACH_IMAGE:latest .
+    - docker build -f docker/bot.Dockerfile -t $BOT_IMAGE:latest .
+    - docker run --rm $BOT_IMAGE:latest make test-stories
+
+
+#############################################################
+############### BUILD REQUIREMENTS ##########################
+#############################################################
+build requirements:
+  stage: build requirements
+  image: docker
+  tags:
+    - docker
+  services:
+    - docker:dind
+  script:
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - cd ./docker/bot
+    - ./build-base.sh publish
+  only:
+    refs:
+      - master
+    changes:
+      - ./docker/bot/requirements.txt
+  environment: homolog
+
+
+#############################################################
+############### BUILD COACH #################################
+#############################################################
+build coach:
+  stage: build coach
+  image: docker
+  tags:
+    - docker
+  services:
+    - docker:dind
+  script:
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - docker build -f docker/coach.Dockerfile -t $COACH_IMAGE:latest .
+    - docker push $COACH_IMAGE:latest
+  only:
+    - master
+  environment: homolog
+
+
+#############################################################
+############### BUILD BOT ###################################
+#############################################################
+build bot:
+  stage: build
+  image: docker
+  tags:
+    - docker
+  services:
+    - docker:dind
+  script:
+    - docker login -u $DOCKERHUB_USER -p $DOCKERHUB_PASSWORD
+    - docker build -f docker/bot.Dockerfile -t $BOT_IMAGE:latest .
+    - docker push $BOT_IMAGE:latest
+  only:
+    - master
+  environment: homolog
diff --git a/Makefile b/Makefile
@@ -10,4 +10,7 @@ run-telegram:
 	docker-compose up telegram_bot
 
 run-console:
-	docker-compose run bot make run-console
+	docker-compose run bot make run-console
+
+test-dialogue:
+	docker-compose run --rm bot make e2e
diff --git a/analytics/setup_elastic.py b/analytics/setup_elastic.py
@@ -5,7 +5,8 @@
 from elasticsearch import Elasticsearch
 
 parser = argparse.ArgumentParser(description='configures elastic')
-parser.add_argument('--task', '-t', default='setup', choices=['setup', 'delete'],)
+parser.add_argument('--task', '-t', default='setup',
+                    choices=['setup', 'delete'],)
 args = parser.parse_args()
 
 logging.basicConfig(level=logging.DEBUG)
@@ -21,18 +22,19 @@
     "mappings": {
         "message": {
             "properties": {
-                "environment":       { "type": "keyword" },
-                "version":           { "type": "keyword" },
-                "user_id":           { "type": "keyword" },
-                "is_bot":            { "type": "boolean" },
-                "text":              { "type": "text" },
-                "tags":              { "type": "keyword" },
-                "timestamp":         { "type": "date", "format": "yyyy/MM/dd HH:mm:ss" },
-                "intent_name":       { "type": "keyword" },
-                "intent_confidence": { "type": "double" },
-                "entities" :         { "type": "keyword" },
-                "utter_name":        { "type": "keyword" },
-                "is_fallback":       { "type": "boolean" },
+                "environment":       {"type": "keyword"},
+                "version":           {"type": "keyword"},
+                "user_id":           {"type": "keyword"},
+                "is_bot":            {"type": "boolean"},
+                "text":              {"type": "text"},
+                "tags":              {"type": "keyword"},
+                "timestamp":         {"type": "date",
+                                      "format": "yyyy/MM/dd HH:mm:ss"},
+                "intent_name":       {"type": "keyword"},
+                "intent_confidence": {"type": "double"},
+                "entities":         {"type": "keyword"},
+                "utter_name":        {"type": "keyword"},
+                "is_fallback":       {"type": "boolean"},
             }
         }
     }

diff --git a/bot/actions/actions.py b/bot/actions/actions.py
@@ -1,15 +1,12 @@
 from rasa_core_sdk import Action
-from rasa_core_sdk.events import SlotSet
-import requests
-import random
+
 
 class ActionTest(Action):
-   def name(self):
-      return "action_test"
+    def name(self):
+        return "action_test"
 
-   def run(self, dispatcher, tracker, domain):
+    def run(self, dispatcher, tracker, domain):
         try:
-          dispatcher.utter_message("Mensagem enviada por uma custom action.")
+            dispatcher.utter_message("Mensagem enviada por uma custom action.")
         except ValueError:
-          dispatcher.utter_message(ValueError)
-
+            dispatcher.utter_message(ValueError)
diff --git a/bot/fallback.py b/bot/fallback.py
@@ -6,7 +6,6 @@
 import logging
 import os
 import json
-import io
 import typing
 
 from typing import Any, List, Text
@@ -107,8 +106,9 @@ def predict_action_probabilities(self, tracker, domain):
         # to not override standard behaviour
         nlu_confidence = nlu_data["intent"].get("confidence", 1.0)
         # logger.debug("NLU_DATA [intent] = {}".format(nlu_data["intent"]))
-        #core_confidence = tracker.current_state()
-        #logger.debug("This should be core_confidence {}".format(core_confidence))
+        # core_confidence = tracker.current_state()
+        # logger.debug("This should be core_confidence {}"
+        # .format(core_confidence))
 
         # if tracker.latest_action_name == self.fallback_action_name:
         #     result = [0.0] * domain.num_actions
@@ -132,18 +132,22 @@ def predict_action_probabilities(self, tracker, domain):
             # the fallback action will be executed.
             result = self.fallback_scores(domain, self.core_threshold)
             try:
-                if tracker.latest_action_name != nlu_data["intent"].get('name'):
-                    if nlu_data["intent"].get('confidence') > self.core_threshold:
-                        idx = domain.index_for_action("utter_{}".format(nlu_data["intent"].get('name')))
-                        result[idx] = nlu_data["intent"].get('confidence',self.nlu_threshold)
-            except Exception as e:
+                if (tracker.latest_action_name !=
+                        nlu_data["intent"].get('name')):
+                    if (nlu_data["intent"].get('confidence') >
+                            self.core_threshold):
+                        idx = domain.index_for_action("utter_{}".format(
+                                                      nlu_data["intent"]
+                                                      .get('name')))
+                        result[idx] = (nlu_data["intent"]
+                                       .get('confidence',
+                                       self.nlu_threshold))
+            except Exception:
                 pass
             # logger.debug("else result= {}".format(result))
             # result = [0.0] * domain.num_actions
             # result[idx] = FALLBACK_SCORE
             # logger.debug("FALLBACK_SCORE = {}".format(FALLBACK_SCORE))
-
-
         return result
 
     def persist(self, path):
@@ -160,7 +164,6 @@ def persist(self, path):
 
     @classmethod
     def load(cls, path):
-        # type: (Text) -> FallbackPolicy
         meta = {}
         if os.path.exists(path):
             meta_path = os.path.join(path, "custom_fallback_policy.json")

diff --git a/bot/test_stories.py b/bot/test_stories.py
@@ -158,4 +158,4 @@ def print_evaluation_statistics():
     else:
         run_evaluation(stories_path, False, None, use_e2e)
 
-    print_evaluation_statistics()
+    print_evaluation_statistics()
diff --git a/bot/tracker_store.py b/bot/tracker_store.py
@@ -12,7 +12,7 @@
 
 try:
     from nltk.corpus import stopwords
-except Exception as e:
+except Exception:
     import nltk
     nltk.download('stopwords')
     from nltk.corpus import stopwords
@@ -25,11 +25,13 @@
 BOT_VERSION = os.getenv('BOT_VERSION', 'notdefined')
 HASH_GEN = hashlib.md5()
 
+
 def gen_id(timestamp):
     HASH_GEN.update(str(timestamp).encode('utf-8'))
     _id = HASH_GEN.hexdigest()[10:]
     return _id
 
+
 class ElasticTrackerStore(InMemoryTrackerStore):
     def __init__(self, domain,
                  user=None, password=None, scheme='http', scheme_port=80):
@@ -53,10 +55,14 @@ def save_user_message(self, tracker):
             '%Y/%m/%d %H:%M:%S'
         )
 
-        #Bag of words
+        # Bag of words
         tags = []
-        for word in tracker.latest_message.text.replace('. ',' ').replace(',',' ').replace('"','').replace("'",'').replace('*','').replace('(','').replace(')','').split(' '):
-            if word.lower() not in stopwords.words('portuguese') and len(word) > 1:
+        for word in (tracker.latest_message.text.replace('. ', ' ')
+                     .replace(',', ' ').replace('"', '')
+                     .replace("'", '').replace('*', '')
+                     .replace('(', '').replace(')', '').split(' ')):
+            if (word.lower() not in stopwords.words('portuguese')
+                    and len(word) > 1):
                 tags.append(word)
 
         message = {
@@ -79,8 +85,8 @@ def save_user_message(self, tracker):
         }
 
         self.es.index(index='messages', doc_type='message',
-                 id='{}_user_{}'.format(ENVIRONMENT_NAME, gen_id(ts)),
-                 body=json.dumps(message))
+                      id='{}_user_{}'.format(ENVIRONMENT_NAME, gen_id(ts)),
+                      body=json.dumps(message))
 
     def save_bot_message(self, tracker):
         if not tracker.latest_message.text:
@@ -98,8 +104,6 @@ def save_bot_message(self, tracker):
                     evt = tracker.events[index]
                 utters.append(evt.action_name)
             index -= 1
-
-
         time_offset = 0
         for utter in utters[::-1]:
             time_offset += 100
@@ -147,8 +151,8 @@ def save_bot_message(self, tracker):
             }
 
             self.es.index(index='messages', doc_type='message',
-                     id='{}_bot_{}'.format(ENVIRONMENT_NAME, gen_id(ts)),
-                     body=json.dumps(message))
+                          id='{}_bot_{}'.format(ENVIRONMENT_NAME, gen_id(ts)),
+                          body=json.dumps(message))
 
     def save(self, tracker):
         if ENABLE_ANALYTICS:

diff --git a/coach/train.py b/coach/train.py
@@ -12,12 +12,13 @@ def train_dialogue(domain_file,
                    training_folder,
                    policy_config):
     return train(domain_file=domain_file,
-                                      stories_file=training_folder,
-                                      output_path=model_path,
-                                      policy_config=policy_config,
-                                      kwargs={'augmentation_factor': 50,
-                                              'validation_split': 0.2,}
-                                      )
+                 stories_file=training_folder,
+                 output_path=model_path,
+                 policy_config=policy_config,
+                 kwargs={'augmentation_factor': 50,
+                         'validation_split': 0.2, })
+
 
 if __name__ == "__main__":
-    train_dialogue('domain.yml', '/src_models/dialogue', 'data/stories/', 'policy_config.yml')
+    train_dialogue('domain.yml', '/src_models/dialogue',
+                   'data/stories/', 'policy_config.yml')
diff --git a/requirements.txt b/requirements.txt
@@ -5,5 +5,6 @@ sklearn-crfsuite==0.3.6
 nltk==3.3
 rocketchat-py-sdk==0.0.8
 elasticsearch==6.3.1
+flake8==3.7.8