diff --git a/docker-compose.yml b/docker-compose.yml index 2bccc02a..96126e02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -73,7 +73,20 @@ services: env_file: - docker/notebooks.env volumes: - - ./bot:/work/bot + - ./coach:/work/coach - ./notebooks:/work/notebooks + - notebook_models:/models:ro ports: - 8888:8888 + depends_on: + - coach + + # =============================== Coach ================================= + coach: + image: botcoach:latest + volumes: + - notebook_models:/notebook_models + command: sh -c "cp -r /src_models/* /notebook_models" + +volumes: + notebook_models: diff --git a/docker/notebooks.env b/docker/notebooks.env index 3bc123ee..61af9af3 100644 --- a/docker/notebooks.env +++ b/docker/notebooks.env @@ -1,8 +1,8 @@ -BOT_DIR_PATH=../../bot/ -BOT_DOMAIN_PATH=../../bot/domain.yml -BOT_NLU_CONFIG_PATH=../../bot/nlu_config.yml -BOT_STORIES_PATH=../../bot/data/stories/ -BOT_INTENTS_PATH=../../bot/data/intents/ -BOT_MODELS_PATH=../../bot/models/ -BOT_MODELS_NLU_PATH=../../bot/models/nlu/current/ -BOT_MODELS_DIALOGUE_PATH=../../bot/models/dialogue +COACH_DIR_PATH=../../coach/ +COACH_DOMAIN_PATH=../../coach/domain.yml +COACH_NLU_CONFIG_PATH=../../coach/nlu_config.yml +COACH_STORIES_PATH=../../coach/data/stories/ +COACH_INTENTS_PATH=../../coach/data/intents/ +COACH_MODELS_PATH=/models/ +COACH_MODELS_NLU_PATH=/models/nlu/current/ +COACH_MODELS_DIALOGUE_PATH=/models/dialogue \ No newline at end of file diff --git a/docs/README-en.md b/docs/README-en.md index 2e3d7d91..345a907b 100644 --- a/docs/README-en.md +++ b/docs/README-en.md @@ -16,7 +16,7 @@ Various technologies are used in boilerplate that interact with each other to ob The user interacts with Boilerplate via RocketChat or Telegram, which sends the messages to the Rasa NLU via -connectors, where it identifies the * intent *, and responds to the Rasa Core, according to *stories* and *actions*. +connectors, where it identifies the *intent*, and responds to the Rasa Core, according to *stories* and *actions*. The *models* used for the conversation were generated by the *trainer* module and then transferred to the bot, these models can be versioned and evolved between bots. Notebooks evaluate the operation according to the format of *intents* and *stories*. diff --git a/notebooks/intents/.ipynb_checkpoints/intents-analysis-checkpoint.ipynb b/notebooks/intents/.ipynb_checkpoints/intents-analysis-checkpoint.ipynb index ad9c062e..c15cddbf 100644 --- a/notebooks/intents/.ipynb_checkpoints/intents-analysis-checkpoint.ipynb +++ b/notebooks/intents/.ipynb_checkpoints/intents-analysis-checkpoint.ipynb @@ -7,541 +7,487 @@ "id": "kO9wt2g3okLS" }, "source": [ - "# Analisando o bot\n", + "# Análise das Intents\n", "\n", - "Este jupyter-notebook vai auxiliar na análise de detecção de intenções da Taís." + "Este jupyter-notebook vai auxiliar na análise de detecção de intenções de seu chatbot." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "## Instalação\n", - "\n", - "### Configurando jupyter" + "### Configurando e Imports" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { - "scrolled": true + "scrolled": false }, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rasa_nlu: 0.15.0\n" + ] + } + ], "source": [ - "%matplotlib inline\n", - "\n", - "import logging, io, json, warnings\n", - "logging.basicConfig(level=\"INFO\")\n", - "warnings.filterwarnings('ignore')\n", + "from IPython.display import IFrame\n", "\n", - "def pprint(o):\n", - " # small helper to make dict dumps a bit prettier\n", - " print(json.dumps(o, indent=2))" + "import rasa_nlu\n", + "print(\"rasa_nlu: {}\".format(rasa_nlu.__version__))" ] }, { "cell_type": "markdown", - "metadata": {}, + "metadata": { + "colab_type": "text", + "id": "1xeXgpdwzOAl" + }, "source": [ - "### Listando versões" + "### Treinando o modelo do Rasa NLU\n", + "\n", + "* Para avaliar o bot o primeiro passo é treiner o seu chatbot. Mas não é necessário treinar a parte de conversão completa (rasa_core) apenas a parte de interpretação de mensagens (rasa_nlu).\n", + "\n", + "* O comando `train-nlu` do Makefile executa o treinamento apenas do `rasa_nlu`" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "rasa_nlu: 0.13.7\n" + "make: Entering directory '/work/coach'\n", + "python3 -m rasa_nlu.train -c nlu_config.yml --fixed_model_name current \\\n", + "--data data/intents/ -o /src_models --project nlu --verbose\n", + "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.train' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.train'; this may result in unpredictable behaviour\n", + " warn(RuntimeWarning(msg))\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/geral.md is md\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'elogios', 'negar', 'despedir', 'diga_mais', 'tudo_bem', 'out_of_scope', 'cumprimentar'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/aleatorio.md is md\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 197 (24 distinct intents)\n", + "\t- Found intents: 'bff', 'filhos', 'license', 'de_onde_voce_eh', 'triste', 'religiao', 'playlist', 'signo', 'risada', 'time', 'comida', 'star_wars', 'cor', 'onde_voce_mora', 'esporte', 'piada', 'me', 'historia', 'linguagens', 'filme', 'hobby', 'relationship', 'como_estou', 'genero'\n", + "\t- entity examples: 172 (22 distinct entities)\n", + "\t- found entities: 'filhos', 'starwars', 'license', 'triste', 'live', 'religiao', 'signo', 'cor', 'how', 'esporte', 'piada', 'me', 'historia', 'hobby', 'relationship', 'bff', 'playlist', 'comida', 'filme', 'linguagens', 'genero', 'where'\n", + "\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/actions.md is md\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 6 (1 distinct intents)\n", + "\t- Found intents: 'action_test'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 288 (32 distinct intents)\n", + "\t- Found intents: 'filhos', 'license', 'negar', 'action_test', 'triste', 'religiao', 'signo', 'time', 'star_wars', 'cor', 'elogios', 'esporte', 'piada', 'me', 'historia', 'hobby', 'relationship', 'tudo_bem', 'como_estou', 'cumprimentar', 'bff', 'despedir', 'de_onde_voce_eh', 'playlist', 'risada', 'comida', 'onde_voce_mora', 'linguagens', 'filme', 'diga_mais', 'out_of_scope', 'genero'\n", + "\t- entity examples: 172 (22 distinct entities)\n", + "\t- found entities: 'filhos', 'starwars', 'license', 'triste', 'live', 'religiao', 'signo', 'cor', 'how', 'esporte', 'piada', 'me', 'historia', 'hobby', 'relationship', 'bff', 'playlist', 'comida', 'filme', 'linguagens', 'genero', 'where'\n", + "\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:30:03 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:03 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "\n", + "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", + "For more information, please see:\n", + " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", + " * https://github.com/tensorflow/addons\n", + "If you depend on functionality not listed there, please file an issue.\n", + "\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use keras.layers.dense instead.\u001b[0m\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Colocations handled automatically by placer.\u001b[0m\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use keras.layers.dropout instead.\u001b[0m\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.cast instead.\u001b[0m\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n", + "2019-07-17 16:30:06.320809: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:30:06.344961: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:30:06.345414: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x563c2cae7710 executing computations on platform Host. Devices:\n", + "2019-07-17 16:30:06.345467: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:30:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:14<00:00, 25.65it/s, loss=0.101, acc=0.993]\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.101, train accuracy=0.993\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Successfully saved model into '/src_models/nlu/current'\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished training\n", + "make: Leaving directory '/work/coach'\n" ] } ], "source": [ - "import rasa_nlu\n", - "\n", - "print(\"rasa_nlu: {}\".format(rasa_nlu.__version__))" + "!make train-nlu -C $COACH_DIR_PATH" ] }, { "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1xeXgpdwzOAl" - }, + "metadata": {}, "source": [ - "### Treinando o modelo do Rasa NLU" + "### Métodos de avaliação do chatbot\n", + "\n", + "* O Rasa fornece vários métodos de avaliação e validação das `intents`, para verificar como utiliza-los, cada método fornece um log, imagem, gráfico ou arquivo com dados relevantes para interpretação do chatbot." ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "Na celula abaixo todas as intents na pasta `../data/intents/` serão utilizadas para gerar a matrix de confuzão para de avaliação do bot.\n", + "* O comando `!python -m rasa_nlu.test` é a base para a avaliação do chatbot.\n", + "\n", + "* Na célula abaixo a flag `-h` foi utilizada para mostrar as funções e a forma de uso de cada uma delas, mude seus valores e flags para ter as informações desejadas na sua análise.\n", "\n", - "Ela irá treinar o modelo e executar a avaliação que pode ser verificada na saída da célula abaixo." + "* Atualmente o Rasa possui 2 modos, `evaluation` e `crossvalidation` que tem seções decicadas a eles neste jupyter notebook." ] }, { "cell_type": "code", "execution_count": 3, "metadata": { - "colab": { - "autoexec": { - "startup": false, - "wait_interval": 0 - }, - "base_uri": "https://localhost:8080/", - "height": 377 - }, - "colab_type": "code", - "executionInfo": { - "elapsed": 2821, - "status": "ok", - "timestamp": 1521380062879, - "user": { - "displayName": "Alexander Weidauer", - "photoUrl": "//lh6.googleusercontent.com/-j4xnI5_PFWA/AAAAAAAAAAI/AAAAAAAAAAo/wPT2w5Bl3xg/s50-c-k-no/photo.jpg", - "userId": "100444450157165144272" - }, - "user_tz": -60 - }, - "id": "qHsAH49OePcM", - "outputId": "f840c42d-acf2-4fbf-b439-e5a239ffb32b", - "scrolled": false + "scrolled": true }, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "INFO:rasa_nlu.training_data.loading:Training data format of ../../bot/data/intents/random.md is md\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 237 (30 distinct intents)\n", - "\t- Found intents: 'me', 'religion', 'its_ok', 'playlist', 'where_u_liv', 'license', 'weather', 'towel', 'laugh', 'genre', 'book', 'sons', 'relationship', 'name', 'sport', 'color', 'food', 'story', 'how_am_i', 'languages', 'sad', 'movie', 'team', 'star_wars', 'where_r_u_from', 'joke', 'bff', 'sign', 'creators', 'hobby'\n", - "\t- entity examples: 208 (28 distinct entities)\n", - "\t- found entities: 'religion', 'how', 'genre', 'book', 'relationship', 'name', 'ok', 'story', 'sad', 'movie', 'where', 'weather', 'bff', 'creators', 'hobby', 'me', 'playlist', 'license', 'towel', 'sons', 'sport', 'color', 'food', 'starwars', 'languages', 'joke', 'sign', 'live'\n", - "\n", - "INFO:rasa_nlu.training_data.loading:Training data format of ../../bot/data/intents/lappis.md is md\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 169 (14 distinct intents)\n", - "\t- Found intents: 'salic_api', 'meu_signo', 'software_livre', 'quem_eh_lappisudo', 'promova_cultura', 'como_contratar', 'pessoa_mais_cool', 'chatbot_tais', 'quais_tecnologias', 'o_que_eh_lappis', 'quais_projetos', 'zoacao_java', 'empurrando_juntos', 'salic_ml'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "INFO:rasa_nlu.training_data.loading:Training data format of ../../bot/data/intents/general.md is md\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 125 (10 distinct intents)\n", - "\t- Found intents: 'despedir', 'tudo_bem', 'negar', 'erro_resposta_utter', 'diga_mais', 'quem_eh_voce', 'cumprimentar', 'out_of_scope', 'elogios', 'afirmar'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 531 (54 distinct intents)\n", - "\t- Found intents: 'software_livre', 'religion', 'its_ok', 'where_u_liv', 'quais_tecnologias', 'cumprimentar', 'genre', 'book', 'relationship', 'name', 'salic_api', 'meu_signo', 'story', 'negar', 'out_of_scope', 'sad', 'movie', 'chatbot_tais', 'diga_mais', 'where_r_u_from', 'weather', 'bff', 'empurrando_juntos', 'creators', 'hobby', 'salic_ml', 'me', 'despedir', 'quem_eh_lappisudo', 'como_contratar', 'playlist', 'pessoa_mais_cool', 'license', 'towel', 'quem_eh_voce', 'laugh', 'elogios', 'sons', 'zoacao_java', 'afirmar', 'sport', 'color', 'food', 'tudo_bem', 'promova_cultura', 'how_am_i', 'languages', 'erro_resposta_utter', 'team', 'star_wars', 'o_que_eh_lappis', 'quais_projetos', 'joke', 'sign'\n", - "\t- entity examples: 208 (28 distinct entities)\n", - "\t- found entities: 'religion', 'how', 'genre', 'book', 'relationship', 'name', 'ok', 'story', 'sad', 'movie', 'where', 'weather', 'bff', 'creators', 'hobby', 'me', 'playlist', 'license', 'towel', 'sons', 'sport', 'color', 'food', 'starwars', 'languages', 'joke', 'sign', 'live'\n", - "\n", - "INFO:rasa_nlu.model:Starting to train component tokenizer_whitespace\n", - "INFO:rasa_nlu.model:Finished training component.\n", - "INFO:rasa_nlu.model:Starting to train component ner_crf\n", - "INFO:rasa_nlu.model:Finished training component.\n", - "INFO:rasa_nlu.model:Starting to train component ner_synonyms\n", - "INFO:rasa_nlu.model:Finished training component.\n", - "INFO:rasa_nlu.model:Starting to train component intent_featurizer_count_vectors\n", - "INFO:rasa_nlu.model:Finished training component.\n", - "INFO:rasa_nlu.model:Starting to train component intent_classifier_tensorflow_embedding\n", - "INFO:rasa_nlu.classifiers.embedding_intent_classifier:Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:20<00:00, 14.71it/s, loss=0.143, acc=0.976]\n", - "INFO:rasa_nlu.classifiers.embedding_intent_classifier:Finished training embedding policy, loss=0.143, train accuracy=0.976\n", - "INFO:rasa_nlu.model:Finished training component.\n", - "INFO:rasa_nlu.model:Successfully saved model into '/work/notebooks/intents/models/nlu/default/current'\n" + "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\r\n", + " warn(RuntimeWarning(msg))\r\n", + "usage: test.py [-h] [--debug] [-v] -d DATA [--mode MODE] [-c CONFIG]\r\n", + " [-m MODEL] [-f FOLDS] [--report [REPORT]]\r\n", + " [--successes [SUCCESSES]] [--errors ERRORS]\r\n", + " [--histogram HISTOGRAM] [--confmat CONFMAT]\r\n", + "\r\n", + "evaluate a Rasa NLU pipeline with cross validation or on external data\r\n", + "\r\n", + "optional arguments:\r\n", + " -h, --help show this help message and exit\r\n", + " --debug Print lots of debugging statements. Sets logging level\r\n", + " to DEBUG\r\n", + " -v, --verbose Be verbose. Sets logging level to INFO\r\n", + " -d DATA, --data DATA file containing training/evaluation data\r\n", + " --mode MODE evaluation|crossvalidation (evaluate pretrained model\r\n", + " or train model by crossvalidation)\r\n", + " -c CONFIG, --config CONFIG\r\n", + " model configuration file (crossvalidation only)\r\n", + " -m MODEL, --model MODEL\r\n", + " path to model (evaluation only)\r\n", + " -f FOLDS, --folds FOLDS\r\n", + " number of CV folds (crossvalidation only)\r\n", + " --report [REPORT] output path to save the intent/entitymetrics report\r\n", + " --successes [SUCCESSES]\r\n", + " output path to save successful predictions\r\n", + " --errors ERRORS output path to save model errors\r\n", + " --histogram HISTOGRAM\r\n", + " output path for the confidence histogram\r\n", + " --confmat CONFMAT output path for the confusion matrix plot\r\n" ] } ], "source": [ - "import os\n", - "\n", - "from rasa_nlu.training_data import load_data\n", - "from rasa_nlu.model import Trainer\n", - "from rasa_nlu import config\n", - "\n", - "\n", - "intents_directory = '../../bot/data/intents/'\n", - "\n", - "intents = {}\n", - "\n", - "for intent_file in os.listdir(intents_directory):\n", - " intent_file_path = os.path.join(intents_directory, intent_file)\n", - "\n", - " intents[intent_file] = {}\n", - "\n", - " intent_list = []\n", - " intent_name = None\n", - "\n", - " with open(intent_file_path) as f:\n", - " lines = f.readlines()\n", - "\n", - " for line in lines:\n", - " line = line.strip()\n", - "\n", - " if line.startswith('##'):\n", - " if intent_name is not None:\n", - " intents[intent_file][intent_name] = intent_list\n", - " intent_name = line.replace('## intent:', '') \n", - " intent_list = []\n", - "\n", - " elif line.startswith('- '):\n", - " intent_list.append(line.replace('- ', ''))\n", - "\n", - "\n", - "# loading the nlu training samples\n", - "training_data = load_data(intents_directory)\n", - "\n", - "# trainer to educate our pipeline\n", - "trainer = Trainer(config.load(\"../../bot/nlu_config.yml\"))\n", - "\n", - "# train the model!\n", - "interpreter = trainer.train(training_data)\n", - "\n", - "# store it for future use\n", - "model_directory = trainer.persist(\"./models/nlu\", fixed_model_name=\"current\")" + "!python -m rasa_nlu.test -h" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "* O comando abaixo gera informações relevates para a validação das `intents` são elas:\n", + " * Matriz de confusão\n", + " * Histograma\n", + " * Erros de intents" ] }, { "cell_type": "code", "execution_count": 4, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [ { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "INFO:tensorflow:Restoring parameters from /work/notebooks/intents/./models/nlu/default/current/intent_classifier_tensorflow_embedding.ckpt\n", - "INFO:rasa_nlu.training_data.loading:Training data format of ../../bot/data/intents/random.md is md\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 237 (30 distinct intents)\n", - "\t- Found intents: 'me', 'religion', 'its_ok', 'playlist', 'where_u_liv', 'license', 'weather', 'towel', 'laugh', 'genre', 'book', 'sons', 'relationship', 'name', 'sport', 'color', 'food', 'story', 'how_am_i', 'languages', 'sad', 'movie', 'team', 'star_wars', 'where_r_u_from', 'joke', 'bff', 'sign', 'creators', 'hobby'\n", - "\t- entity examples: 208 (28 distinct entities)\n", - "\t- found entities: 'religion', 'how', 'genre', 'book', 'relationship', 'name', 'ok', 'story', 'sad', 'movie', 'where', 'weather', 'bff', 'creators', 'hobby', 'me', 'playlist', 'license', 'towel', 'sons', 'sport', 'color', 'food', 'starwars', 'languages', 'joke', 'sign', 'live'\n", - "\n", - "INFO:rasa_nlu.training_data.loading:Training data format of ../../bot/data/intents/lappis.md is md\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 169 (14 distinct intents)\n", - "\t- Found intents: 'salic_api', 'meu_signo', 'software_livre', 'quem_eh_lappisudo', 'promova_cultura', 'como_contratar', 'pessoa_mais_cool', 'chatbot_tais', 'quais_tecnologias', 'o_que_eh_lappis', 'quais_projetos', 'zoacao_java', 'empurrando_juntos', 'salic_ml'\n", + "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\n", + " warn(RuntimeWarning(msg))\n", + "2019-07-17 16:31:33.800517: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:31:33.828980: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:31:33.829499: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55c71b5187e0 executing computations on platform Host. Devices:\n", + "2019-07-17 16:31:33.829554: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:31:34 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use standard file APIs to check for files with this prefix.\u001b[0m\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mtensorflow\u001b[0m - Restoring parameters from /models/nlu/current/component_4_EmbeddingIntentClassifier.ckpt\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/geral.md is md\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'cumprimentar', 'diga_mais', 'out_of_scope', 'despedir', 'tudo_bem', 'negar', 'elogios'\n", "\t- entity examples: 0 (0 distinct entities)\n", "\t- found entities: \n", "\n", - "INFO:rasa_nlu.training_data.loading:Training data format of ../../bot/data/intents/general.md is md\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 125 (10 distinct intents)\n", - "\t- Found intents: 'despedir', 'tudo_bem', 'negar', 'erro_resposta_utter', 'diga_mais', 'quem_eh_voce', 'cumprimentar', 'out_of_scope', 'elogios', 'afirmar'\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/aleatorio.md is md\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 197 (24 distinct intents)\n", + "\t- Found intents: 'relationship', 'me', 'time', 'bff', 'religiao', 'license', 'esporte', 'filhos', 'risada', 'historia', 'de_onde_voce_eh', 'piada', 'genero', 'linguagens', 'cor', 'signo', 'onde_voce_mora', 'playlist', 'filme', 'star_wars', 'triste', 'comida', 'como_estou', 'hobby'\n", + "\t- entity examples: 172 (22 distinct entities)\n", + "\t- found entities: 'me', 'bff', 'religiao', 'license', 'piada', 'cor', 'filme', 'triste', 'comida', 'where', 'relationship', 'how', 'live', 'esporte', 'historia', 'filhos', 'genero', 'linguagens', 'signo', 'starwars', 'playlist', 'hobby'\n", + "\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/actions.md is md\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 6 (1 distinct intents)\n", + "\t- Found intents: 'action_test'\n", "\t- entity examples: 0 (0 distinct entities)\n", "\t- found entities: \n", "\n", - "INFO:rasa_nlu.training_data.training_data:Training data stats: \n", - "\t- intent examples: 531 (54 distinct intents)\n", - "\t- Found intents: 'software_livre', 'religion', 'its_ok', 'where_u_liv', 'quais_tecnologias', 'cumprimentar', 'genre', 'book', 'relationship', 'name', 'salic_api', 'meu_signo', 'story', 'negar', 'out_of_scope', 'sad', 'movie', 'chatbot_tais', 'diga_mais', 'where_r_u_from', 'weather', 'bff', 'empurrando_juntos', 'creators', 'hobby', 'salic_ml', 'me', 'despedir', 'quem_eh_lappisudo', 'como_contratar', 'playlist', 'pessoa_mais_cool', 'license', 'towel', 'quem_eh_voce', 'laugh', 'elogios', 'sons', 'zoacao_java', 'afirmar', 'sport', 'color', 'food', 'tudo_bem', 'promova_cultura', 'how_am_i', 'languages', 'erro_resposta_utter', 'team', 'star_wars', 'o_que_eh_lappis', 'quais_projetos', 'joke', 'sign'\n", - "\t- entity examples: 208 (28 distinct entities)\n", - "\t- found entities: 'religion', 'how', 'genre', 'book', 'relationship', 'name', 'ok', 'story', 'sad', 'movie', 'where', 'weather', 'bff', 'creators', 'hobby', 'me', 'playlist', 'license', 'towel', 'sons', 'sport', 'color', 'food', 'starwars', 'languages', 'joke', 'sign', 'live'\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 288 (32 distinct intents)\n", + "\t- Found intents: 'me', 'cumprimentar', 'out_of_scope', 'bff', 'religiao', 'license', 'negar', 'elogios', 'risada', 'de_onde_voce_eh', 'piada', 'cor', 'filme', 'triste', 'comida', 'como_estou', 'relationship', 'action_test', 'despedir', 'hobby', 'esporte', 'filhos', 'historia', 'genero', 'linguagens', 'signo', 'diga_mais', 'playlist', 'tudo_bem', 'onde_voce_mora', 'time', 'star_wars'\n", + "\t- entity examples: 172 (22 distinct entities)\n", + "\t- found entities: 'me', 'bff', 'religiao', 'license', 'piada', 'cor', 'filme', 'triste', 'comida', 'where', 'relationship', 'how', 'live', 'esporte', 'historia', 'filhos', 'genero', 'linguagens', 'signo', 'starwars', 'playlist', 'hobby'\n", "\n", - "INFO:rasa_nlu.evaluate:Intent evaluation results:\n", - "INFO:rasa_nlu.evaluate:Intent Evaluation: Only considering those 531 examples that have a defined intent out of 531 examples\n", - "INFO:rasa_nlu.evaluate:F1-Score: 0.9747239685377217\n", - "INFO:rasa_nlu.evaluate:Precision: 0.9778755247681801\n", - "INFO:rasa_nlu.evaluate:Accuracy: 0.9736346516007532\n", - "INFO:rasa_nlu.evaluate:Classification report: \n", - " precision recall f1-score support\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 288/288 [00:00<00:00, 429.64it/s]\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent evaluation results:\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent Evaluation: Only considering those 288 examples that have a defined intent out of 288 examples\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - F1-Score: 0.9914054566832343\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Precision: 0.9942129629629629\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Accuracy: 0.9895833333333334\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Classification report: \n", + " precision recall f1-score support\n", "\n", - " 0.00 0.00 0.00 0\n", - " afirmar 0.87 1.00 0.93 13\n", - " bff 1.00 1.00 1.00 11\n", - " book 1.00 1.00 1.00 5\n", - " chatbot_tais 1.00 1.00 1.00 10\n", - " color 1.00 1.00 1.00 6\n", - " como_contratar 1.00 1.00 1.00 19\n", - " creators 1.00 0.83 0.91 6\n", - " cumprimentar 1.00 0.91 0.95 22\n", - " despedir 1.00 1.00 1.00 16\n", - " diga_mais 1.00 1.00 1.00 9\n", - " elogios 1.00 1.00 1.00 4\n", - " empurrando_juntos 1.00 1.00 1.00 6\n", - "erro_resposta_utter 1.00 1.00 1.00 6\n", - " food 1.00 1.00 1.00 8\n", - " genre 1.00 1.00 1.00 8\n", - " hobby 1.00 1.00 1.00 7\n", - " how_am_i 1.00 1.00 1.00 8\n", - " its_ok 1.00 0.90 0.95 10\n", - " joke 1.00 1.00 1.00 7\n", - " languages 0.93 0.93 0.93 14\n", - " laugh 1.00 1.00 1.00 13\n", - " license 1.00 0.91 0.95 11\n", - " me 1.00 1.00 1.00 7\n", - " meu_signo 0.94 0.94 0.94 17\n", - " movie 1.00 1.00 1.00 4\n", - " name 1.00 1.00 1.00 4\n", - " negar 1.00 1.00 1.00 19\n", - " o_que_eh_lappis 0.77 1.00 0.87 10\n", - " out_of_scope 1.00 1.00 1.00 5\n", - " pessoa_mais_cool 1.00 1.00 1.00 19\n", - " playlist 1.00 1.00 1.00 8\n", - " promova_cultura 1.00 1.00 1.00 8\n", - " quais_projetos 1.00 1.00 1.00 20\n", - " quais_tecnologias 1.00 1.00 1.00 17\n", - " quem_eh_lappisudo 1.00 0.89 0.94 9\n", - " quem_eh_voce 0.90 0.90 0.90 21\n", - " relationship 1.00 1.00 1.00 10\n", - " religion 1.00 1.00 1.00 16\n", - " sad 1.00 1.00 1.00 7\n", - " salic_api 1.00 1.00 1.00 10\n", - " salic_ml 1.00 1.00 1.00 10\n", - " sign 0.67 0.67 0.67 3\n", - " software_livre 0.89 1.00 0.94 8\n", - " sons 1.00 1.00 1.00 11\n", - " sport 1.00 1.00 1.00 6\n", - " star_wars 1.00 1.00 1.00 10\n", - " story 1.00 1.00 1.00 6\n", - " team 1.00 1.00 1.00 7\n", - " towel 1.00 1.00 1.00 4\n", - " tudo_bem 0.82 0.90 0.86 10\n", - " weather 1.00 1.00 1.00 8\n", - " where_r_u_from 1.00 0.75 0.86 4\n", - " where_u_liv 1.00 1.00 1.00 8\n", - " zoacao_java 1.00 0.83 0.91 6\n", + " 0.00 0.00 0.00 0\n", + " action_test 1.00 1.00 1.00 6\n", + " bff 1.00 1.00 1.00 11\n", + " comida 1.00 1.00 1.00 8\n", + " como_estou 1.00 1.00 1.00 8\n", + " cor 1.00 1.00 1.00 6\n", + " cumprimentar 1.00 0.91 0.95 22\n", + "de_onde_voce_eh 1.00 1.00 1.00 4\n", + " despedir 1.00 1.00 1.00 16\n", + " diga_mais 1.00 1.00 1.00 9\n", + " elogios 1.00 1.00 1.00 4\n", + " esporte 1.00 1.00 1.00 6\n", + " filhos 1.00 1.00 1.00 11\n", + " filme 1.00 1.00 1.00 4\n", + " genero 1.00 1.00 1.00 8\n", + " historia 1.00 1.00 1.00 6\n", + " hobby 1.00 1.00 1.00 7\n", + " license 1.00 1.00 1.00 11\n", + " linguagens 1.00 0.93 0.96 14\n", + " me 1.00 1.00 1.00 6\n", + " negar 1.00 1.00 1.00 19\n", + " onde_voce_mora 1.00 1.00 1.00 8\n", + " out_of_scope 1.00 1.00 1.00 5\n", + " piada 1.00 1.00 1.00 7\n", + " playlist 1.00 1.00 1.00 8\n", + " relationship 1.00 1.00 1.00 10\n", + " religiao 1.00 1.00 1.00 16\n", + " risada 1.00 1.00 1.00 13\n", + " signo 1.00 1.00 1.00 3\n", + " star_wars 1.00 1.00 1.00 10\n", + " time 1.00 1.00 1.00 5\n", + " triste 1.00 1.00 1.00 7\n", + " tudo_bem 0.83 1.00 0.91 10\n", "\n", - " avg / total 0.98 0.97 0.97 531\n", + " micro avg 0.99 0.99 0.99 288\n", + " macro avg 0.96 0.96 0.96 288\n", + " weighted avg 0.99 0.99 0.99 288\n", "\n", - "INFO:rasa_nlu.evaluate:There were some nlu intent classification errors. Use `--verbose` to show them in the log.\n", - "INFO:rasa_nlu.evaluate:Model prediction errors saved to errors.json.\n", - "INFO:rasa_nlu.evaluate:Confusion matrix, without normalization: \n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Model prediction errors saved to errors.json.\n", + "2019-07-17 16:31:35 \u001b[1;30mINFO \u001b[0m \u001b[34mmatplotlib.font_manager\u001b[0m - font search path ['/usr/local/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf', '/usr/local/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/afm', '/usr/local/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/pdfcorefonts']\n", + "2019-07-17 16:31:36 \u001b[1;30mINFO \u001b[0m \u001b[34mmatplotlib.font_manager\u001b[0m - generated new fontManager\n", + "2019-07-17 16:31:38 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Confusion matrix, without normalization: \n", "[[ 0 0 0 ... 0 0 0]\n", - " [ 0 13 0 ... 0 0 0]\n", + " [ 0 6 0 ... 0 0 0]\n", " [ 0 0 11 ... 0 0 0]\n", " ...\n", - " [ 0 0 0 ... 3 0 0]\n", - " [ 0 0 0 ... 0 8 0]\n", - " [ 0 0 0 ... 0 0 5]]\n" + " [ 0 0 0 ... 5 0 0]\n", + " [ 0 0 0 ... 0 7 0]\n", + " [ 0 0 0 ... 0 0 10]]\n", + "Figure(2000x2000)\n", + "Figure(1000x1000)\n", + "2019-07-17 16:31:43 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity evaluation results:\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Evaluation for entity extractor: CRFEntityExtractor \n" ] }, { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAcgAAAFwCAYAAAA8OOvyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzsnXl4VdXVxn8r80BIgABhHsI8g4gKDoiWClVRPxRxQv0stdXYtDjUoRURsM7WsZWPOiCCFesE4oCWWVGgKCAgYECZwhzGzOv7Y52b3IQbciHDPSHnfZ79cM8+++7z3p2Qdfd+1yCqigcPHjx48OChJMJCTcCDBw8ePHhwIzwD6cGDBw8ePASAZyA9ePDgwYOHAPAMpAcPHjx48BAAnoH04MGDBw8eAsAzkB48ePDgwUMAeAbSg4daCBG5XER+FpFDItK7AvOsFpGBlUgtZBCR2SIyKtQ8PLgHnoH0UGshIptE5MIgx84VkVsq8dkqIu3KGdNERCaLyHYROSgia0XkIRGJrwQKTwC3q2odVf3vyU6iql1VdW4l8KkyiMhYEXmjvHGqOkRVX6sOTh5qBjwD6cGDCyEi9YEvgVjgLFVNAH4BJAGplfCIVsDqSpinxkMM3t9CD8fA+6Xw4AEQkRtFZKGIPCEi+0QkQ0SGOPcmAOcAzztHks87/Z1E5DMR2Ssi60TkKr/5XhWRF0RklrP7WyIiqc69+c6wb535RgSg9EfgIHCdqm4CUNWfVfX3qvqdM09/EflGRLKcf/v7PX+uiDwsIouc538qIskiEi0ih4Bw5/kbnfEldrQO//HO62QRmSki+53PusBnUPx34c7cz4jINqc9IyLRzr2BIrJFRMaIyE5nV3zTcX4ec0VkvIgsdtboQxFpICJTReSA83lb+43/m3NkfEBElonIOU7/RcB9wAhnnm/95p8gIouAI0Bb/1MCEXlJRN7xm/9REflcRKQszh5OPXgG0oOHYpwBrAOSgceAySIiqno/sIDiI8nbnWPOz4A3gUbA1cCLItLFb76rgYeAesAGYAKAqp7r3O/pzPdWAC4XAv9W1cJARJ0d5izgWaAB8BQwS0Qa+A27BrjJ4RcF3KmqOapax+/5wexGxwBbgIZAY8zgBMpReT9wJtAL6An0Ax7wu58CJALNgP8FXhCResd57tXA9c74VGxH/QpQH1gDPOg39hvnufWxn8nbIhKjqh8DE4G3nLXu6fee64HRQAKwOcBn7u58cTrH4TtKvdyctQqegfTgoRibVXWSqhYArwFNMIMQCBcDm1T1FVXNd3S8d4Ar/ca8q6pfq2o+MBX7Ax4sGgDbj3P/V8B6VZ3iPH8asBa4xG/MK6r6g6oeBf51gs/3Rx62Fq1UNU9VF5RhKK4FxqnqTlXdhX05uL7UPOOcOT4CDgEdj/PcV1R1o6pmAbOBjao6x1nPt4Ei5yJVfUNV9zhr8SQQXc7cAK+q6mrnPXn+N1T1iMP9KeANIE1Vt5Qzn4dTDJ6B9OChGDt8L5w/kAB1yhjbCjjDOXbcLyL7MQOREmg+7BivrLkCYQ9mlMpCU47d9WzGdluV8Xx/PI7tgD8VkR9F5E9Bctrs9PmwxzFuwXLK9Ht9NMB10XtF5E4RWeMcN+/HdqrJx5kb4Ofj3VTVJcCPgGBfMDzUMngG0oOH4FB6x/QzME9Vk/xaHVX9bSU9bw5w+XGcR7ZhRtofLYGtJ/m8I0Cc33WRoVfVg6o6RlXbApcCfxSRC4Lg1NLpq1I4R6B3A1cB9VQ1CcjCDBsEPg4+Xr9v3tuwneg2Z34PtQyegfTgIThkAm39rmcCHUTkehGJdNrpItL5JOcrjaeAusBrItIKQESaichTItID+Mh5/jUiEuE4+nRxeJ0MVgDXiEi449hynu+GiFwsIu0cB5UsoAAIpI1OAx4QkYYikgz8BTuerGokAPnALiBCRP6CrZ0PmUDr43zZOAYi0gEYD1yHHbXeLSIne0TtoYbCM5AePASHvwHDxTxcn1XVg8BgzJFkG3ac+Si24wgGYzHjt1/8vF99UNW9QH9Mt1siIgeBzzEDtUFV92A66BjsOPZu4GJV3X2Sn+/3mH7pOyp+z+9ee2xHewhzlHlRVf8TYI7xwFLgO2AlsNzpq2p8AnwM/IAd62ZT8vj0beffPSKyvLzJRCQCM+yPquq3qroec0ya4vPK9VBLoKpeq+UNuAjz3twA/Kn0daAx1dHnFh4et5rNw+NWeTxqWws5Aa+F+BfA4uE2Ysd9UcC3wE+lrrsFGFMdfW7h4XGr2Tw8bpXDo0uo/15Vd/OOWD30w47sflTVXGAxkO13PR24rdSY6uhzCw+PW83m4XGrPB7DqGXwDKSHZpTUa/Kc5sMWoHWpMdXR5xYeHreazcPjVnk8/EOIagVE1UsMUZshIsOBW6ZNm3bWhRdeGL9///6c9u3bb1LVrsDDe/bsGbV3796kWbNm5dx1111Z+fn5AMswL0EB2mDHtJXdlwk0xxwuqmJ+j1vt4OFxqzweu1T1dmoTKnI+67Wa34ABwJGZM2eO2LFjR7/vv/8+H8sQg6rWBe4F/p6YmJj79ddfT8X0iO3AWzg5LKuobzf2Dbaq5ve41Q4eHrfK4/F0qP9eVfvfx1AT8FpoW0JCgoqIduvWTYcOHaorV67UsLAw7datm/bu3VtjY2O1VatW2rZtW50+fbr27t1bIyIiNCkpqeh9VdEXHh6uQJXN73GrHTw8bpXHAzhS0b83YQktVWIbBtWAVVjYkK+Nru6/j94Ray1HamqqtmvXjoyMDFJSUnjjjTe47bbbWLduHQUFBbz++uts376dDz/8kLVr17J792569+7NoUOHUFUyMjIoKCio9L6GDRuybds2YmJiqmR+j1vt4OFxqzwe77zzzi5VbVSRvzdhcY01ulOg4jXHIvu/zy1T1b4VeV5F4TnpVCNE5EonX+R/RKSviDwbIh6jRWSpiCw9cOAAIoKIsG3bNiZPngzAI488wtdff02HDh0AiIyMZMmSJWzcuJG3336b9u3bF71PRAgLC/PNXSV9VT2/x6128PC4VYxHhSGASHDNDQj1EV9tali2j7PLGRNRyc887nwdOnTQ2NhY3bhxo+bk5Gjnzp2ViFi98K6p2v/Of+vK1Ws16ozROnjwYPVh06ZNOn78+BLvS0lJ0eHDh1daX/369TUiIqLK5ve41Q4eHrfK4wHsqOjfI4lrpDGnpQfVgKWh/psdETLLfIpDRN4DWgAxWJqyFOBsrMbgB1gtvztV9WIRGYvVu2sL/CQinwCXAfFYmq8nMKH8eiAHGKqqe0Xk11g9uygs28X1qnpERF7FvM96A4uw4rvl8QWgV69erN20ky835XJOl3pIZCx6aDfrd64nIyODZs2asXPnTtq1a1fifYHmqkhf0X+oKprf41Y7eHjcKp9HheGW3WEQ8DTIKoKI1HeMWCxWzPU8rF7gnaq6VEQGUtJAXoLtLo+KyI1YodnemIHdANyjqn8XkaexuoXPiEgDtZyciFV/z1TV5xwDmQwMU6ttWCZ8GmRaWhr9+vVj3LhxfPrZHC5Mn8xlZ7aid4s47hhzN9ePuIL09HQKCgr44osv+Oabb5g0adIpr8V43Go2D4+byzTI+BSN7npdUGOzv3nS0yBPYdwhIt8CX2E7yfbljP9ArbCtD/9RKzO0C0tQ/aHTv43iKu3dRGSBiKwEbgYeEpEVWNxSGPCdiDxe+kESQIMcM2YM/fv3Z/ny5az/eSd/v3s4Fw/sy8RHH2fGogzCwsKK9Ihp06YBsHjxYqKjo0lISGDZsmW+uU9pLcbjVrN5eNxcoEGGhQfX3IBQn/Geig0YCCwE4pzruU7fXKCv35iZzuux2G7S9/4bgef9rjcByc7rO7HCswAZQE/n9X+A+c7rV4HDQHh5XEtrkKW1hy5dumhaWpqGJzTWBlc9r41uelPbdeyi3YffqzExMbpr1y5VVZ04caKOHj36lNNiPG41m4fHzWUaZHyKxpx5T1ANF2iQ3g6yapAI7FPTAzsBZ1by/GEiMhXbmY4XkXRM3+zt9A/CjmaXidUJLBcigbWHIUOGkJmZSXjdFCLqNkbCI7lg6OUc2beD/Px8Nm/eTG5uLtOnT2fAgAEl5go0f7B9pXlUZK7K7vO41RweHrfK51FhSFhwzQXwNMgqgFjNuPew/IbrgCRslziWsjXIQ6r6hPP+G7Gd5u3O9SbnereI3Ak8jhnEHsBjwD5sx7hDVc8X0yBHqGpseVz94yD9tYcXX3yRfv36MXv2bKZNm8aWozGcPuJuJozsxcfvvc2iL79k1gfvsX//flSVIUOGcN1111WaLukWLcbjVrN5eNxcpkHWaaLRPW8Oamz24okh1yA9L9YqgKrmAEMC3BroN2YuduQKdiQ6E/NWRVVfdfp8Y1v7zTEDuENVFwGLRGQdcAdW6HaoiCSr6o1iOVYDQkRGY96vJCcnB9QexowZQ0FBAX369AHgh42bWHvfcKbcW0hEg7aExdbl5See4LbbbuPQoUPMmjWLPn360KJFC6ZNm8aIESNYuHAh+fn59OzZ85TRYjxuNZuHxy3EGiTimt1hUAj1Ga/XFGynueoExm72ux4EvIsZ1J0Ua5WHgpmvPA2yLM0iLS1N6zRqrufc/qgOeegNlbBwHTlypA4ePFjvuusufeSRR3TixInar1+/k9JU3KLFeNxqNg+Pm8s0yDpNNObsPwfV8DRID36IEJGpYpl2ZohInIhcICL/FZGVIvJPsaNbgJYisk7Me/Ul4Ev/icRCS2LE4iSDgsiJaSBbt26lTsOmtDnrIiJi4kCEyMhI1q9fz4wZMxg5ciTTp0/n2muvpaCgoNz5TpZHKPo8bjWHh8et8nlUDAJhEcE1F8DTIF0AEWmNeaSeraqLROSfwI/Ab4ALVPUHEXkdWA7MxpL4fgR0AOKA54GuwFCsAPL/OXPFlPfssjTI8jSQXbt28dWyFcQ3SCE/5yjZB/Yy+uYbGTp0KJdccgmtW7fm5ptvpkOHDlxzzTW0adOmRmoxHreazcPj5jINMqGZRve9Naix2XP/EnIN0ttBugc/q+mKAG8AFwAZqvqD0/cacC7mnfqlqg5T1c5Y/ONZqnojVq7mfeCV4xlHKSMX64loILGxsTRo242CvBwKC/KL5vYfHx4eXvSeHj16EBkZyeLFi9m/f3+N1WI8bjWbh8fNBXGQNciLNaTnu25pmH43/ATGJwG/87seiBPTeAJz3Of3ujXH6oozgfV+fRcA/wZ64sQ7+vX74iI3AS8CU3BOB8prJ6tB+vetW7dOIyIiNL77xRpeN0XDExppkxsmaWSD1vr8889reHi4Tpo0SVVVx40bp6NGjaoxWozHrWbz8Li5TINMaKoxgyYE1fA0yBqLJOB3FZzjvlLXLUXkLOf1NcAaoJWItHP6rgfmYWEjrUv1j/Ob5y9Y2McLJ0JGpGIaiKpSeGQfkYkpxLXrz5H184lrfw4zZsygsLCQQYMGkZuby4wZMxg0aFBQ850Mj+ro87jVHB4et8rnUTFIjdpB1koNUkRuwDLSKPAdUAAcAPpiScXvVtUZIlIHO7KsB0QCD6jq+yIyHRiGGavPsMTj44CDQDssq83vVLVQREZixlCAWap6j4j8FbgLWAmsBu7HKn0sBU4DvgcKgUsdbocx7bE5ZpyTnI9yGMvzepWq1hGRn4GfMV2yFTBbVa853lqcrAbp61u8eDFHjx6lsLAQlXCiGncgeei97PnkcfL2bCY5wVLR7dy5k5ycHLp37056ejpTpkypEVqMx61m8/C4uUyDrNtco8/8fVBjsz+7O+QapDtchaoRItIVy2XaXy3wvj7wFNAEC77vBHyAxRtmA5er6gERSQa+EqvE8Segm6r2cuYciDnHdAE2Y8buChFZDDyKGb19wKcicpmq/klEbve930GnUjxbA51VtZtzHYEZvj9ixvMGoIeqqohc5bztGSBGVSeISLgzPtAalBsHGaw+0bRp06J4yQ+XbEBRdr5zD2gh0c26cV7vprzzzjtMmzaNCy64gKysLO6++25atGjBJ598AsC+fftYt24dEyZMcL0W43Gr2Tw8bqHWIMVaTUGoz3iruwFpwIRSfa8C1/pdH3T+jcQ8RL8DVgBHsR1ma/ziFjEN0l8XvBkzVsOA1/36/xd4ynl93DjFAM/wcckEtvq4+M+FOfFswDL29HL6TqgeZGVrIM36X6oSFq53vDpPb5v+X139/Rrt2bNnifqSEydO1FdeeUUbNWrkOi3GzTqRm7m5hYfHzWUaZN3mGvPLJ4NqeBqkq5Dj99r3FedaoCFwmtpuLxPzIg2E0mfVJ3R2LSI3iMh3IvKtiEzBjlNbO32fY4a9IfAP4GmHSx8R+QqIFZF3sSPbc4GRwGcikgEEdZ4hUjUaSEH2YcKjY1mRkcnhfJDwSBISEli/fj2FhYVFeVyjoqLo0KFDlfGozD6PW83h4XGrfB4Vhm8XWV5zAWqdBil2xPouFhqxR4qPWGeq6gxnzCE1Te/3QDtVTROR84EvgDaY1rhcVVs54wdiGqHviHU28DKwGCt35Tti/QR4Tk3H3Ac0UtU8P07+x75vYvlXk0XkZiAdc9LZAzQGbgXWOv/Ocj5DM+zo9HMshGe5qqYfbz0qqkGW1zdvzVbCI6M5ui+TFk0a86/XXubpJx7n8ssvJyUlhZdffpkWLVrQsWNHPvroIxYuXOgqLcbNOpGbubmFh8fNZRpkYkuNHjAmqLHZs9NDrkHWuh2kqq4GJgDzxOo1PnWc4VOBvmIZa27ADBJqRYoXicgqKa63+A12BLoGC/p/V1W3Y3rlf4BvgWXOc3+HGdDvpLj6xtuqutuZfy/mMPSZiKzCkgA0w45sf4sZ3B+Auqo6z3n+a1jIx7fO/STgb4E+lFRCHOSJ9BXk5lCQfYSfMn7klVdfJzo6mrCwMEaOHMmcOXMID7fab2+//XZRfckff/yx0nlUtT4Tah5VwW3VqlXMnz+fqKgoNm7cyLBhw7w1qoXcKk2DtElrzA4ypOe7tbERIO8qji6Kn14I7AYindeRzvUhnIogWEmtn/zGp2I7RihZd/K4NSGrWoMs3delSxcNa9hLJTpRo7rfpNF97lCJTdapU6eWqC85evRojYyMDLkW42adqDq4nX/++XrmmWdq165dVTX4up+1aY1ORW5VpkEmttSYi18IquFpkCcOKaXViUhrEfnC6ftcRFo6414VkZdE5CsR+VFEBorlM10jVg7KN99IsVynq0Tk0XKefZGILHee/bnTV19E3nOe/5WI9HD6xzrPm+s8/w5nmr8CqSKyQkQeFzuevRHTCtc6752FeRhniHmcXgvsBWKxI9VrVDULSBTLyboacwry7SbPAdLFdsi+2Mry1hWoeg1kyJAhkH8YiU6icUpzJCyc8PqdyMjIKFFfcs6cOYSFhVUZj4r0VfUauYlbt27d2LJlC3l5eSdU97M2rdGpyO14PCqMGrSDrFEapATW6l4DZqjqa2Ja3aWqepljBGMwh5VLsewyA7C4w28wj9KdlNQIPwWeVdX3Ajy7IZYL9VxVzRCR+qq6V0SeA3ar6kMiMgjzUu0lVuNxMHA+kIDFTKZgR6UztTh841LnM20HGjiPy8Y8Y/+GFVtegiUE+B4riXVIVZ8QkXOxepDxWNxjL1X9UUQUuFdV/1remla1BunrS0tLK6ov+cprU8jMKuRX16Txm+EDeO/fb7N54xpmzJhRVF+yZ8+e1KtXL+SakJt1ourglpKSwmmnncatt95K8+bNufnmm+nYsWO5dT9r0xqdityqTINMaqXR55XOkRIY2R/cGnINsqbFQR6j1Ylln7nCuT8FMxg+fKiqKqYhZqrqSgBnx9UaMypzVXWX0z8V8wI9xkBihmq+qmb4nu30nw38j9P3hYg0EJG6zr1ZarUhc0RkJ+ZcUxoHnHnP93WIyDNYLOZ+zLv2HlX9SURQ1bGl1iMWi4tUoBGW5LwAK6ocEFKJcZDB9vnXl1ywbAMA6+7/X566vxCJa8RvrxnMiy++SLNmzahfvz75+fksWLCA0aNHA7B9+3YKCwuZOXMmTz31lCv1mVDzqCpu/vd8erG3Ru7jdvToUdavX09SUhKRkZHExMRw6NChSuVRKRB37A6DQqjPeE+kETiG8Ritznn9Kk5+VY6NKXwVGM5x4hQDPPsSYGqA/v8Cbf2ufwbqUqwVRjj9qxwepbkMxC+Pq3O9EvjIuZ4LDHReHyo1biEQ51zPCzSuvFbdGmRZfR0Hj9TUbn10zBsL9NrXlmr9xk1148aN6sMXX3yho0eP1oYNG7pSnzlVNazf3Z6mLVq01Hbt22vW4Rzt3r2HTpkyxVsjF3K75JJL9Mwzz1RV1QMHDmiDBg3cp0EmtdKYyyYF1fA0yBPGF8CVItIATP/DQimudu5fCyw4gfm+Bs4TkWSxzDMjKdbxSuMr4FwRaeP3bJznPS6mQW7AjjufBS7GDPpjIhKPHa2+h+0MGzlztHbGniOmbfbHnG+aAwNEZA12LBwlIq9g8Y7/FQs5SXSeNV0sFvIcoL6IzHfGrRKRc4JdCJHQaiBH9mZyWCPYJXUpzM8jO6+QNWvWsH37dlSV8ePHM2DAAHJycmjevHm1civdF6o1CgW3zB2ZtGzViqioKKKiorhyxNUsWLCg2nlUZt+pyq1169ZkZmaSkZFBdHQ0eXl5tGnTptJ5VBjiaZBVBhEZheUxLcB2bw8CrwDJwC7gJrXjyFdxYhsdQ+Sv+/nfOyZX6nGePQSYiIXH7FTVX4gd8X6GxT8exHaNtwD9gUmq+riITMQ0xHOwY9OfsJ3mp1jM5B3AH4BpmEGcj8VUfo6Fa3yP6am7sJ1vCpYabyHQ2Zkn3hm3FbgXM6BxqnrweOtZXRpkeX0LVmew/+cNxDdIobAgn659z+K9SU8zePBg5s2bx3PPPVekfb399tv07t3bVfrMqaphLVy0iANZWeTl5dGocWMGD76I7du2eGvkQm4pKSkMHTqU9PR0srOzOXDgAM8++yxTp051jQYZXq+1xlzwYFBjj7xzs6dBnihU9TXMMccfgwKMu9Hv9SagWxn3pmGGKZhnz8YMmj/6ApOBZqo6HEBEbsGOgn08BwNZFGub+zFDtw2LnWwFvA10UNUcEbkHuFNVL3Pmexf4F/aF5h4RWYAd1T4FnKeqNznjzgX+iTnydFfVFYE+h4RAgyyvLyIqhgZtu3F491YoLOTMTi0ZP348R44c4ZNPPiE9PZ38/Hw+/vhj6taty8aNG/n6669p0qQJ8+bN4+GHH64ROlFN49ajR09UYfNm+8N58OABb41czC0sLAxVZefOnQwfPpy4uLhK5VFhiNNqCkJ9xlvTGxaeESi363C/62VAxwDvHYsZszDsy0q+0z+Qkrrku9gOc6ZzvQDogYWHPF9qzqbAr7HcsTeWx98tGmTpvjZt2ihhURrV+Trd+FOmnj7gAl21NqNEDtcff/xRf/e739Uonai2cHMLj9rELS0tTZu2aK09+56lt98zVtt16qpjn/iHqzTIsHqtNf7KV4JquECDrHE7yONBApexCpRCbiDwELaT647tzlZixi7WeR9AS8xDNA4LCbldVWeKyI2Y52wdLISjgYhcoaqdReQ2bEfbW0SewAzgPmChiGwEhmI7xgOYgQsHzgA+AsLFjn+jMV3yR+BuzCDeC0SL6ZKNsVCQe4F6Ytrl7djxaiqW9ScMy9n6apBrB4Ren2nYsCHh4eFkZWVBeBRh0Ym8/8V3pHbsjmoB69evJyMjg5YtW/LII49w9tln88orr1Qr31CvUU3g5hYetYlbZmYmuTnZdOrWk+tHp1NQUMh/l35Z6TwqiqqYs6pQ4zTIsiCBYyTLyrE6EDvu7IwF4P8I/J+qPiiWf7WNqqY7xioFM2qpWMq4dphT0His3NReERmDZcJZh+mQPbCCyp9g1TUewI54/wfIw3aUscAjDsdYIAqLg5yBGd762G6wAdDC+WwDgI1YyaslmFNSTyxechrwHLYrbek8Y4Q6YSllwS0a5KFDh4iKiuKDDz5g7dq1PP744/xn3gJ6DEnnm5WbueOyNlw7/BK+++5b0tPTee+999izZw+ZmZm89957HD16lBUrVrheJ6ot3NzCozZx27JlC0uWLKFdxy5IWBgHsvaRVC+ZVs0au0eDrN9G4wc/FNTYg2+N8jTISkSgGMnjjf9GLVcqzs7uU6d/JRbc78O/VLUQWO/s6Hx1Gz/T4ljId4B7sDjM3cAAVX3DmTsL+LeqPiciSzDj+QDmcPMEtkM9jB2vtnKM8ruqOtV5/0FVzRbL8nOnql7s9CdiO9BOWCxmF2znfBPwoPrFVZaGuFCDBMjNzaVjx44UFBTQqFEjMn7azqZJ94MW8nhGW371ywuLsutceuml3HLLLbRr144NGzZw6NAhYmJiiI6OrrTYrxPVZ6rzmTWNm1t4nMrckpKSOPOcQWzb8hOFBQV063U6hw8drFQeFYaAhNWcHWRIz3crsxE4RvL/gKuc12FArvN6ICU1vrkU5y4tuocdT97kN24+tmO7ET/tD3OY2Ykd75a+twlIdl7fiDnl1AW2l/E5XqWkfnmoDM5jMYeh5zm+fhnSepCVpcU0a9ZM586dqxJbX6P636UzF63ROsktNSL1lxofH6+HDx/WvLw8TU1N1REjRrhWJ3KzhnUq8vC4uSsOMrx+G028ZkpQDRdokDUtDvJ4CBQjuQlLIweWbi7yJOa9UkTCRCQVaIsdo+I8434R+QHzQI1yuhsCl4jIMjFv00hn7JXYsezVwEwsz+oLIvK+WL7WzSLyoN/c14nI11hM4z+wXWaCiNzkPPNWoJ4z/HpMv7zTuT5LRJ4RkaWEuB7kyfaV5hEXF8cbb7yBxDYgLLYBH3+9mf7nD6Hw4HZEhMzMTAoLC4uOiqqTW2XPfypwcwsPj1vl86go/Heqx2tuwCmjQQLIsTGS9wDvYxrfx8BtWqxB+h9XznWul/rfEzvuzMZCOeoCf9RiJ50h2LHmGZjhXAbcD9wMLFPV60XkDGx32gLTLydjGuYDmHH7ADsi/RF4C3P8yQDmABc61/uA1ynOH9sPGAf82+k76sy0nmZiAAAgAElEQVT7Byyecyl25PuWqv6uvDVzkwbp3+fP48knn2THjh2888471EtuTGKvq3lzziqu65nP5rVLWbH0K2JiYigsLKSgoICuXbtSWFjoSp3IzRrWqcjD4+auXKwRDdpq3aHjgxq7741rPQ2yMqGBYyTP9Ht9jzNuLma4fO8b6Pe6xD1gjqreWuo5r4pIErBOVY8Aq0TkBSyYvw2QKyK+GMRNak5Di4BfYR6z4WoJz58EBqnqDVD07W0vZuBPwwzgBqzO405Ms7xCVcc74+/FYifvEZGjPv5iuWffKmudxKUaZFl9f/jDH+jTpw8A/5r9JfruR6DKP/7bFImqQ9pVV/HSSy8hIvTs2ZOwsDCaNWvGRx99RN++fdmwYQMpKSlVxtcNa+R2bm7h4XELsQZJ4J2raxHqM143N0rmcx2LaYzjsN1dOjDOb+xTwF8oQ1t0xpzhvH8T5p16I/Ca3/1x2JFoGvBIgPdfRsncsXdgO9MbfPyc/rk4mmp5raZokMeL/YpMTNHWv/6ntvvjBxoel6gJXX+hgwcP1ieffFJHjhypHTt2DKqGYU3TidzMzS08PG4u0yAbtNH6N7wZVMPTIN0NVb1RnRARv76/qOoczGHnMhGJFZEELJn5EUxbvBJADD2d16mqukRV/4J5sLZwpvyFWE3JWMwALsJSzA0XEV/O1voi0goL7ThPrGJIJHAlsFpVX6/oZ/V9q/P9YgT6lledfcHyyMzMJKJuIyKTmpB/eB+qSkRCA9assfJZo0aNYtu2bUHVMKxsbqHocws3t/DwuFU+j4pAsN1oMM0NOKU0yMqGiNwPjMKON3/GdMZuFOdxfQ1LcJ6LBekXYMeob2LhHGGYMfwFFvPYHhDMAKY7c9+BGct4rLTVnzGHnzQsqfkmTAf9xJmnMaZfrsPqU/bCjm3rOK87YHGQi1T1l2V8Lv8j1tP69OlTIzSQst43Z8ESwmMTICyC+DZ9KczLplvcXn766Sfy8/OpW7cu48ePL7eGYU3TidzMzS08PG7u0iAjk1O13qWPBDV21ysjPA3SrRCR0zCP017YOi3HDKTvfgwWe9lRTU+cBiQ4r38JHFHVfBG5EJioqlc479sEPKyq6nw724s5ALXFNMwNWP3HNiLyNLBZVZ8R887Nx3auA7H6ls+JFWZGVceKyDagjVo+16SyPpuqvgy8DOakU5M0kEBjohu1JS8rEwqyycvaQWFeDklNG7Jr1y4OHTpU9LkbNWpEQkICq1atYvHixbRs2ZKkpKRTRidyMze38PC4hV6D9HKxngKNwBrjnRTXkuwFzPO7fynF8ZO+zDersMQDa/3GbaJkXOQcrOqH7/5PWOJzMI/YZ5zX52Ep51Zinq5/d/rHUqw9foxl4rkOqBPM56zpGmSgvtNOO00lLFwTGjTWuKQGioj2799fmzRpopMmTVJV1XHjxumoUaNqtE7kZm5u4eFxc5cGGZHcVhv977+Canga5CmLhzHnmb9ix65tReQfYjUn/VEfC/O4QkTSnb5C4HYRWQf8CRgqFt/4KrbbfMhpbUTkv8BvgatEJBo73s3FUs/tFvOSDQoiNUcDKa8vNTWV1r3OIm3KfC7/09PUb9qaQYMGsWvXLgYNGkRubi4zZsxg0KBB1c6tqvvcws0tPDxulc+jovDfqR6vuQGeBlkGRKQPZpTOoPiI9R84GiQwC/gBOEdVN4nIVCBRLX7yXcxA/gKLx7wBS0b+Feap2hdLWP4qlj/1NCxd3HXOvHuA07Ed5l+xBAN/wnab72L64ZlAb+AarDjzVOeZbzkcNwNnqermAJ/tlNIgA/Xt0AR+lT6Bzd8t4bN/TGTg6T1YsWIF27dvJycnh+7du5Oens6UKVNqrE7kZm5u4eFxc5cGGdWwnSZf8VhQY7e//D+eBulWqOpyEXkL+BZz0vmm1P2jIvI74GMROVzq/mNYgoJ6mKGrT3Esow9nY8YuF0tg/m+soHIMMFtVs0UkGztOBXPeeQwzpkuBg6r6g/NNaylwLpYMvaXznJnA9jI+2ymlQQbqO7B7B3//9S/RwkIap3ZBVdmwYQPNmzcnMjKS6Oho3nvvvUrRJd2yRm7m5hYeHrfaq0GKSFssmUuiOrV7y0Woz3hrWsMv1yqOzof9yF8E/uA3rkQsI5Ya7gYcDRKLd/TXOB/GPFrTgYf8+n3a530U6589gfl+Yy7AEqKDlcoaihVO/qK8z3MqapCB+pr1v1QlLFwHjvu3XvTcQo1LbqY9e/Y8KV3SLWvkZm5u4eFxc5cGGdkwVZve+u+gGkFokM7fuZ3AqlL9F2Ge/huAP5W6NyNYvqe0BikiEce7rgT8WixjzmogETuC9aFELCMWijHP7/4CLI4yTkTigcudvkVYLtcYEamDHZ+CGUgf1gGtRaSdc309MM8Zn6iqH2Gp53oG+0FEao4GcjJ9BdmHCY+OpbAgn7CISGIaNCE2NvakdEm3rJGbubmFh8et8nlUFP471eO1IPEqZgz95w8HXqA4HehIEelyUlwdi1qlEJHrsN1RFBbs/jsgC3gJ2+1sxwzAY9gRYbqqfiCW8/RyzPg0A95Q1YdEpDXmMdrNmf9ObDc3Viyv6grsCHMaVhA5G9PrFgHTMSeWGCyP6U2qus551qVYceRUrOTU3c78N2HFifdjR645qnq7w+OfWHzjJuBaVf3J73OPcN4X5vAvdD6LL24yFzvmTsUKI+93OPTANMr9WFhHHDAYS1i+QlXPFZGXsN2sOJ/3POcZy7FdpACTVfW2AD+PU16DLN03b81WwiOjObovEy3IR8LCuaBvF1avXn3CuqRb1sjN3NzCw+PmMg2yUTttfGVwvoNbXrwsKA0ygD04CxirThy4WEpOVPUR53qGBnnEWuU7SBHpDIzAaiT2wrw6r8UC47/AjMFBzBHlF5iONs5vin5OXw+sskaZC+a3Q4xS1b6q6vtJNMcKKf8RWIs51vTGUsNN9Juil8O1OzBCRFqISBPMa3QAxXUXfXgOy/26EXOQedafj6q+paq9VLWHM/ZRVY3CNMNrVbU7pjtuU9Vn/Dici32JiMGSm9/vrNs4xzj6nHqSnZbg8OqN1cSMVdUYTLc8Bqr6srM+fevWrVujNJCKzFWQm0NB9hEKcrOJa9iciIgINmzYQL169WjRokWRLpmVlUVkZCSxsbFs3bqVgoKCGrFGbubmFh4eN5dokME0SBaRpX5tdJBPaIYldvFhC9BMLAPZ34He4hjNchHsWezJNsyA5GK7td3Y8WAuFvT+LfaH3rdTWo5lpjmIeXz+jMUF1nPmGodpdK1xzpyxvKPzgW3AGGyXOA9zmvkG8x4dhe2wfMeh+4HvsSoaRzDP0e1OX5gz7wrM8G32e1Y4tgPeg8UjHsIM6iHnc+VjlUP+4jx7FeYMI5SRK9Xhu815fbvz2ac7a6TOmt2D7T63Orz+hiUuf8d5zjbgaSyLzn6K4y+nlffzqS0aZLC6ZJcuXTQmJkaPHDmiqqrdunXT8847z5VrVNM0LG+Nag63qtIgoxq105ZpHwTVCDIO0t8eONfDgf/zu74evxq9J9KqdAcptnvsAzytqrGYDjcBq5GYr6o9VXWhM/yAqvZR1WnYkeI92A5oH/Cg37SKGSJ/7tHAy86OsR2W0Pt0bOfZHzuavBO4DTMo47Ed6X2YQUtzXtfBYhKbAh2BP2JaXgMRuQzb3SVihqc7lhru39iO8AZgv6oexX4Ypztb/liKdUQf/PmX/hmoql6NHYFuwpKlP+q852lnF/4j8Evn+nSsHuVIVf0B00HjMUO5hiAhUnM0kMrqC6RLxsfHk5eXx9q1azly5AibN2/m9NNPr3ZuJ9vnFm5u4eFxq3weFYX/TvV4rQLYSnGua7ATxK0nM1FVH7FegOUOTReRVc51V+y4ML/U2BUAIpKIaaPznP4U4Hwpmcw7E2gkln5NMMPmQz3gbjHnmQ8wYxzjvO8p7BgyS1XzMXE3R1V/xAzvemxHezq2S8wCvnTmGIztZlsB54rIRc6cVzvPHYI52eDwXSJWdmqQ85n9sYniQs4Nj7+ERSig+Oe1AOgMvCAi32HGNEzMaScPE64fxb6cHAMRGe07tti7dy+dO3emY8eOxMfHU6dOHerVq1d0nZyczMGDB0uMqY6+6uCRn32YOiltWDjhOmbffjaHd2yiZ8+eDB48mD59+hAfH098fDxnnHEGLVu2pFOnTtSpU4ewsDBXrFGo1q0m8fC4VR4PKsleVIOB/AZoLyJtRCQK+xv9wclMVNUGUrBivjdgBjEHKxWVG2BsoD6wY9i2wHfAO6q6VFXzsOPWrzFPzR9LPfMmNe2vF7a7ylbVvwK3AAuBF0Xke+zItDRKeC2p6nZscUdiR7HTsG8jt+I4+WCB/0OB34vlaH0R2/l1ByZhBtp/7ieA34plwvHnEAlEicga4DeleC0DxojIVFVdjjkeRWA/w3vVxPNU4NeYwXwH0z2P/YB+GmRCQgLr169n7dq1HD58mMOHD7N///6i66ysLA4ePFhiTHX0VQePDklhFOzexPp1a8g+cohGdWN5a8EaPv7sc9rf8gLd7v2QnfsO8Oabb7J161b2799Pbm4uSUlJ7Nu3L+RrFKp1q0k8PG6Vx6OMv5cnjuA1yPKnshzYXwIdRWSLiPyvs/m5HSvwsAb4l6quPhmqVW0gP8fOg//jGKuBwBVAoar67/r2Y0YFVc0CVorIOc69JOw8ub2qPuR7g6o+q6qp2M7zIVUd69x6G3Oo8eEZtcobqaq6Us2r833sSHUyoCLSBngdM+ILMcNbgBVCDsc8a2/EjGC6qg4BHsASlQ/CHHTuUfNg9RnD3WJhFz5vqQZYqjhUda2q9lBzFPoci4EEM6DhmLPSE1is41zn3rtY/tVrnesPgFdVtZtaMvNeqvoJ5hn8NKapfhvgZ1ICBQUFREVF0bZtW6KiomjcuDEiUnTduXNn9u3bV2JMdfSFikfOzgzCYxOIa9aR8KhYElL7smrVKvLy8jh69Cj5+fkcOnSIiIiIkK+Rm9bNrTw8bpXHg0pKLFOZO0hVHamqTVQ1UlWbq+pkp/8jVe2gqqmqOuFkuVapgVTV7zFD8qlzFPgZ0CSIt44CHscMRzIlvVrLwx1AXxH5ztkl3ur0p4vIKodHHjDb6f8GeB77ppGBhXdsx1K7/QczMstU9X3MO2quc3z7BhbCAXak+XenPwfbNa7CvsF8A5yPpawLtM33301ej/0SzsaOUC8VkaMish5La3e5iKwUkQXYzvkeEfnB+Zy/F5FPMe/XKwnyO1h4eDg5OTlkZGSQm5tLZmZmkbt3bm4ua9asoV69eiXGVEdfqHiERcZQmJdDYV42qkrewT00btyY+Ph4WrRoQZMmTTh48CDh4eEhXyM3rZtbeXjcKo8Hx8piJw6pliPWSkOVh3moX6iDqp6mql+V2j2iqq1Vdbff9QpVPVNVW6hqW1Xdd5z5B6rqUr/r3ao6wnleF1W91elPc3ZbPZxvHTnOWw6o6q9UtaOq3qqqhc74aara3XnPPU7ft44jUS+nzXb633He3wtz7LkY80aNxzTPT1W1n6ruCcC/aDepqqdhXrPnY9rtM2rOTb/BSmT1wDxg56hqV+AqrKxWF+d5C52j1nHYrjcg/DXIgwcP0qFDhyKtIT4+nqSkpKLrxMREEhISSoypjr5Q8YhIaEB4bAKr/noZK8cPIe/ALtq3b09iYiJ5eXns37+fiIgImjVrRqdOnUhISODAgQMsWbKkVq+bW3l43CqPB3aqViEIQlhYcM0NOKUz6YQCqjrBz4D62sls8c8GpjhzfoF50tYto78eFjv5htM/C/P+LYtjkQZZv3591qxZU6b2sHv3bhISEkqMqY6+UPG4qHcrIrL3s2H9OrKPHqZBbBhvz1/F5q07SP3tZDr/6UOOFIRx8OBBcnJyyMnJ4a677iIyMpLdu3fX2nVzKw+PW+XxwELNKgyR4BonHwdZaagxycpF5AVKaosAf1PVV052Tkffm1sBWkGjFP/GmLftT5jzzTJMY3wBO4L+EAtdQUReBQ4AjfDTFEVkIMVH0N9gsZjDRGQklrEojhP4AlST3Myru08L81FVIuvUR8IiiEhowIED28nIyKBZs2ZMnjyZ5s2bs3Xr1pDzdcu6uYWHx63yeVQUJzDnbg1xNQ+v3FU1Q0ROxzTKMzGvVV8ZrSGYXvoZ5oU6GUsykIoFwiZiGYgWA09ioR6zgR9VtauITMHCU7pjnsIfAbep6ovH45Oamqrt2rWrEamuQpUO7JMVP5F/eB+al4OEhRMWE8/A7q1Yv349BQUFJCUl0atXL7Zs2eKlKnMZD4+bu1LNxTTpoK1HBXSuPwbrHr0o5OWuvCPWaoSItMYcdd5X1WxVPYjtFmOwhAZvA00xA3gEq/hxOeYcNErN6SkcM5QzsNCYkc70KzFDegA7as13rgPxKNIgDxw4UEIYd3uqq1DxiIhPApTC/FwiE5IJCwsr+ia8du1azjrrrBNOUVcb1s1tPDxuoU01J1CjNMgTTr3jtZNvmMHazrHlrP4CbA8wfi5mQIf79R3C0ta9hCXo9fWXKK8VbKvNqeZOtq9x48YaHhOvp/1puna87iENi4rVNsN+X26Kutq0bm7h4XFzV6q5mJT22vX+T4NqBJlqriqbt4OsfmQDt4nI9yLyOXAJFutZR0Q2i8i7IlJPRHylqlKBx8VCVPr5zZMKDBCR9SLyayye8jYRGQUgIvVF5D0RGRYMKd9uyPeL4bsONKY6+tzCI1BfXl4e4TF1iGnQlD0r55HUsR85+3eWm6KuOri5Zd3cwsPjVvk8KoQgHXQq+7EnC0+DrEaIHbFuAP4PS0FXD4u9bIolWr8Jy/caiSUkH+Tcuw8rCvoitgt9ArgOi428AfgvFmc5GssduxlzyW4MtFTLLBEQngZ5cty+y9hBz9+/zNIJV9Lyol9zZMePXNqtIZMmTUJE6NatG3/+85+ZNGlSrVw3t/DwuLlLg4xr2kHb3XJct4girHz4F54GWQuRgVX16IAZwWZAuFq86EWYp+tmVfUlR7hVVWeo6nygLpZ4F+BNVR2qFj/6H6CfWjahLVjO2384Y44xjlKOBnngwAH2799PbGwsw4YN873HFRpIqHn4+gpyjrLimVuIiE0ga8NyCvNz+eqrr2jbti1t2rQhLy+P+fPnk5WVRVRUFDExMURHR9fadXMLD49baDVIkBLPOV5zBUJ9xuv2RqlSKpUxF/AmliJvJ5ba7ie/ManAcuf1XOB8v3s/Yd6sYympY74ODHNe34PtIncCl5THqbQG2aBBA23UqJEuW7ZMu3btqhMnTtTRo0e7QgNxs040ZMgQDYuK1U53vK7dH5itkXUbamyzThofH6+HDx/WvLw8TU1N1REjRtSKdXMLD4+buzTI2CYdtOeDc4JqWFGHpX5tdHX//fd2kCGAql6jlnXnMWAOsE+Kc89ej9Wz9GEEgIicjVUhyXL6h4lIjFhFk4FYLCRY2rt0IENVPwyWk+8bW0REBNnZ2Rw+fBhVZfr06QwYMKDEmEDvq+w+3y9odT6zItxyc3NRLY6VDI9LRAvyEREyMzMpLCwsOsaqbm6VOX9N4+Fxq3weFcUJ7CB3q5PQxGkvVzqZ8rj6/sN6CAwx3XA2ttPrj1XyGIbVi/w7FpC/EbgZ0w5nq+ppYk42K4BWqvqTiGwEfgXMUKsTiYjciZXq2ovVyYzAUsadrao/iMgmLNyjJRb8/wCWGGAUFsbxM5YE/TFVneTH+WOsAso16peGLxBKa5C9e/cmJyeHVatWsXXrVh588EE6duxYa7W0E+E279uNJWIlIxOSid+/npiYGAoLCykoKKBr164UFhae8uvmFh4eN3dpkPHNOmqnW/8e1NjlfxnkaZA1BO2BF9Tyn+7HCjG/jlXw6IHFID6oqjuBGLGUcOdgxwLniEgrYKda3tVuvklV9Qk13XAKUFct7+p9FJe6ehUznvWwJABjscoojbGKI39Vq3LibxzjHL6ZZX0YOYE4yPDw8BLXodZAQs3jeNxKx0pGxCdx1VVXsXXrVnbs2EHTpk05cuQIS5cuJS4ujoSEBGbNmsWmTZtO+XVzCw+PW6g1SJAa5MVaree5NbFhuuF6v+t7gAcpWzechGXF+RcW5P8PzOP0Mb/xi/3mvgbLfvMpZmjXAR8798cC9zuvw7BKIb5d/zis9JY/12ux5AHpmH7Zt7zPV1qDbNy4sdatm6iffT5PO3furN2799DJr05xhQZS03SitLQ0jUxM0da//qe2++MHGh6XqAldf6GDBw9WVdX8/HytU6eOjhw58pRbN7fw8Li5S4OMa9pB+47/T1ANLw6yxiDH73UBFrdYFuZju8dWWN3JnliC8QW+Aara33nZGjOQzwHPqxVY/g3FNSWLnq1WZSRPHUuIJQ4unUt3EfCDqj4T7AfzQaQ4xi8xKZEWLVsiYWFcOeJqFi1cUGJMoPdVdp/vF7Q6n1nZ3DIzM4mo24jIpCaWqk6ViIQGrF+/noyMDD7++GMKCwu56KKLqp1bVfe5hYfHrfJ5VBQ1aQfpaZDlQEyDnKnH6oaXA7er6gIRGQskquofnPHzsWLH14nIR0A3oKc6ZbtE5JCq1hGRr7C6j5FYZpx/Yju/WOBH5/VPqvqE//uc158APYA9WEHpZ/y5OnMnAderqs+B5xgEioP8fs0a4uPi2L17N3USEmjapCnNmzcLuQZSE3WiOQuWEB6bAGERxLfpS2FeNm+Ov5X09HS2b9/OwIEDGTVqVKVpvG5ZN7fw8Li5TINs3lG73Racr83X9w0MuQZZY6p5uBCjsCLJcZgxuwlAVTeJfe2a74xbCDTXwDUt/wTciR3LPo15sC7BDHA0llf1GIiVfRkIPAQ8CywRkXk4Ja5EpCPQFXOLPsY4Ou8fDRAdHc22bdtITU1l1apVpKWlsXXrNmLjYomLj6d37z6YThlaXcQtWsyJcotu1Ja8rEwoyCYvawfhcUmEhYUxceJEzj33XFSVBQsW0KJFCz755BMA9u7dS0ZGBhMmTDhl1s0tPDxuodUgBffsDoNCqM94a2MDDjn/DqRkPtVrgNWYztn+OO9/DjtK9V0/DNyBHdlmAmuBLsFwadGihZ555pnatWtXVVUdPXq0RkZGulIDORV0orS0NI2p31RHPPCCnnvpNbp6zTqdOnVqkS6pqjpx4kSdNm2apqSk1Nh1cwsPj5u7NMj4Zh31rEfnBdXwNEgP/lDVN4FLgaPAOhEZJCK+PKwrRWRE6feIlc/6NRbuEYvpl02AjyWIPKwNGzZky5Yt5OXlkZuby5w5cwgLK65UUepZ1dbn+wUNNY/K5paZmUlMcjM+//AdpEUPwmPiycjIKNIlc3NzmT59OvHx8TRs2LDGrptbeHjcKp9HReG/Uz1ecwM8DbIaISL3A1diXqsrMYMWpaptnPttsQB/FZFcrGxVC+AiigsjnwFcjJXEGoYVWo7B8q/WxzLtDAfuxXaU7VT1cFmc+vbtq7fddhu33norzZs3p1+/fkVHfG7TQE4VnWj5bmHfmsVc/Mj7PHp5Z5599GGGDh1Keno66enpXHfddXz66adMnjyZjRs31sh1cwsPj5u7NMg6LTppr99PKn8gsOiuczcDu/26XtZqThbgaZBlQPwcYioLqjoBmODM3UtEIoFPRORbLOYxGrheRPKwsI5sYJqqFgCZYjrj6cB6LJxjNpaO6e9qTjrfAQlYCEg4ZjBHYUnO/T9bkQbZsmXLEnqDb/foVg0k1Dwqg1vuwT1ExCfy2fjrSd02CoCwsDAeeeQRBg4cSE6OOU03a9aM2bNnAyevS9bUNfK4uZNHRSEE3rmWgd0aYiedkJ7vurnh6IShfD7muHOzX98U7Ah2IOb8sxT4ld/9ZUDHE3lO7959tEWLltqufXvNOpyjbdqmanR0tCs1kFNRJ+rSpYuGp16oEt9QL7xrqva/89+6cvXak9Yl3bJubuHhcXOXBlmneUc958mFQTU8DdL9EJE6IvK5iCwX0wGHOf2tRWSNiEwSkdUi8qmIxDr3TheR70RkhTgaotN/o4g87zf3TBEZ6Lx+SSy7zWoRecgZsgCr8bhWbJc5DCuMDJaS7gfgbbGakMOAT4AHReRr59k/iEj7432+w4cP07JVK6KiooiKiuL8QRdQUFAQ8Ftedfb5fkFDzaOquQ0ZMgSyswir04id2dHsO1KARMaetC7plnVzCw+PW+XzqCjCwiSo5gZ4GmQZkOJYxQggTlUPiEgy8BWWyq0VVtuxr6quEJF/AR+o6huOQfy1qn4pIn8FLlaLTbzRGX+784yZwBOqOldE6qvqXhEJx4ofn+4852cgD1AsSUEBFg7yJBbmMR8zjPFAbyzReV3MgG4Chqvq0bI+Z506dbSwsJC8vDwaNW5Mhw4d2bhhvSs1kFNJJ0pLS6Nfv37Mnj2bV15/g8zsGP761HN0aV6PpOgCHhr7l5PSJd2ybm7h4XFzlwaZ0LKT9h3zz6DGzk0fEPI4SG8HWT4EmCim783B6jc2du5lqOoK5/UyoLWIJAEJqvql0//mMROKHArwnKtEZDlW/LgblqS8EbBQVWPU8rReAyxQ1Q+ALCyO8gMs2048FhM5ETOOr2Op6I4xjuKXizUmNpYBZ59Lm7ZtiYmJ4ejRo74xrtRAQs2jsriNGTOG/v37M2vWLBas3sEPGzO4/MKz6NqlIxMffRwwXTIzM5O0tDQ6dOgAmC7pqzu5cOFCli5dSn5+vuvXzS08qoLbqlWrmD9/PlFRUWzcuLFSa6i6bY0qCvHqQZ4ajeJYxRuBt4BI53oT5h3aGr86kVjA/1gse81mv/4evnFYTtYX/eaeg+mJbbDdaD2gKWb83gB6AfP85roUJ26SUnqjw/N553UqFhe5Hhh0vM9ZOhermzWQ2qIT+euSUWf8RmMGPaCERVEeR00AACAASURBVBTpknfddZc+8sgjOnHiRO3Xr18JLm5ZN7fwqA5u559/folY4orUUHXLulWVBpnQopMOenZxUA1Pg6wRSMQqceSJyPnY0WqZUNX9wEEROcPputrv9ibM6CEiLYB+2FHpR9jO9DzsCDUXM5zrgLYiMkEsnd0IIFFsN9scmCaOvomVxGoq5un6sXP/fcxAlwvfNzbfL0agb3DV2ecWHqHg5q9Ldur3CyQmESLjinTJGTNmMHLkSKZPn861115bQjN2y7q5hUd1cOvWrVuJWOKK1FB1y7odj0dFESYSVHMDPA2yDEixBpkMfIilf1sKnIlV64AAOVpVdaxjHCdhCcXnYbrjALHftjeAkZjxqg88qqoficg0rIzWAkxz7KaqzUXkEmAyVv/xA8zgDsFqTS7GcrludN5TF3gNM6TtnPtXqOresj5noFysbtVATnWdqCxdMi4/i+uGD+HK/7mCoUOHcskll9C6dWtuvvlmOnTowDXXXEObNm1ctW5u4VEd3FJSUjjttNOKYolvvvnmk66h6pZ1qyoNsm7LznrmPa8ENfaz288KuQZZZhykWE3DMqGqAfOEnipQJwZSVXcDZ5UxrERtR7/+1UBbx8BOwLRE1L6NXCsiw1T1crE4yKfFHHkKsd3jtVjg/0xnrv8Aj2EGujlQqI6+KSKjgDe12AFogKo+BDwkIrOBCYGMo/jFQSYnJ9coDSTUPKqS25gxYygoKKBPnz4sWL0DUC6/8CwozENzjwCUGB8eHs62bdsoKCggLi4OgOXLlxMfH09sbOwpuUZu5Vb65+ImbpXJo6IQgXCXeKgGhbLOXjHvyZ+cf38udf1TWe/zmoLt4AqAVcAsoGGp++Xpm82B752+PwDbgV1YjUn/OpT++uaNOBqkcz0TGFgeV0+DdD+3devWaUREhMZ3v1jD66ZoeEIjbXLDJI1s0Fqff/55jYuLU1WrLxkTE+OKdatNP7+0tLQSscTdu/fQKVNOroaqW9atqjTIui076ZCXlgTVcIEGWeYOUlVbVMjy1mKo6lsiMlltZ9ca2wV2EwvheBSIFdMRf8CMXw8RmYzpm69hzjyNRGQhFlaSgHmqPgd0EZGrgT864xOlONaxg4h8DURhDj9Bf+0TqTkaSKj7QsFNVSk8so/IxBQik1tzZP184tqfw/vvv09kZCQZGRl8//33qCrh4eG1co1CxS0zM5OWrVqxb99eoqKiuHLE1SxYcHI1VN2ybsfjUVGcwJTJIrLU77raU80FpUE6f5DbqupEEWkONFbVZVXOrgZDijXM1hTXaPwtcAFwkXOvHaZJdsc0yd7AC0AfTGN8FKv3OAerGdkYuB/blR4G3sPiJS8A0rFj0w5qDkWbsOPX+47H09Mg3c1t8eLFHD16lMLCQlTCiWrcgeSh97Lnk8fJ27OZpg0SeOKJJ/jzn//M9u3b6dChA8nJybVqjULNbdGiRWRlZRXFEg8efBHbt21xBTe3aZBJrTrr2fe9HtTYWbf2c68G6YNY5pdI4Fwsxu4I8HfsD7OHE8OFWN7U4QCqukFEbsGcabpgu8TfANtV9WYRuQK4X1UXiUhjYBGmb96BGcoOwHpVPSoiB7Fcrt843/pyMG/YYyCeBlljuDVt2rRIl/xwyQYUZec794AWEt2sG8N+2YvExMSi+pLZ2dmMGzeuqLbk119/TZMmTZg3bx4PP/zwKblG1clt1apV7Nmzp6h+6pQpUzh06P/ZO+/wKKvsj39OJh1CQq/BEAKhBBIQQWEVRQVBsayoi13cxdUVf7gr7lrByir2sq5gARERxYIVFKUILEqVFiAgQXrvJaSc3x/nnWQIKQNJyCR5j899MnPnvu98544Pd875nvM9B8nOgfXr7XA5cGB/pdu30uIgASoSBekPn7bQ+bvIZ+7X8o4NB/ogj2eMI48n/AS4ON+6dsD/CrnHdCwDFqybRzrGby7GwrPLsWbNPTAJuuEni9PlICs2tgbd+h3XX3LZilTt2rWreu23337Tu+66q0rvUWliK82ax0Det7LiIKPPaK2Xj5zn1yAAOEh/fhZkikgQJnWGiNTGMi5dO3n7HrhDTL4OEamF1TrWFZFznLkQEWlbwLWxWK1kb6w2cx7wf5jXOAHYDlwjIh+IabEuEfNO/TKRisOBlPdcIGHLPLDruP6SwRFRpKam5obKhg8fTpcuXU47tkDao9LEVpo1j6WNrTTnisJREhMsi9WfEQhWLAcpIjcDVwGdgHeAa4HHVPXDsodXcU0K5iCDsZKNS7C6xVGq+pqIpGAycdFY2PslVR0lItOB+1R1voh0ABZimauXAY9i4e55WJePa7AfMR0xJZ4crM6ytRbRD9LlICs2th+WbiAkqtZx/SX/cdefWbVqFZ9//jm7du1i27ZtfP755xw5coTFixdXuT0qTWylWfMYyPtWVhxkzbg22uPRsX6t/fT2ToHPQarqeyKyAOPPAK5R1WVFXePacXWU6Tj1kqqahWWf/j3f2sUYx5v/Huf7PN0DrFHVpcBSEWkNTFHVcSISDwzBxAQOcLyH3xRI9b2vuBxkpcJWUH9JEeHyyy/nz3/+MwkJCYSHhzNmzBg6derEmjVraNCgQZXao9LEVlY1j4G8b6XJQYoEhnfol/kTh8W8lruAO4H25R0XrqwDa3R8USGvxXG89utorFNH7mucQj9Il4OsXNhat26teMI1tOsQ/Wp2qlav01RD2l6nPXv21Oeff1779++viYmJpcqbVbQ9Kgm2gmoe3x59ajWPgbxvZcVB1oxrrf3eXeDXoCJwkCLyEDAeE9FuAnwgIg+U1gHtWp6p6qOqOrUEt5gCDBLnJ5pYWNYv8/6q8/6PUdCvvNM5Fyg4Khq2lJQUCA4nKKI2k39ZT9cLepNzZA+pqalMnDiRW265hc2bN5cqb1bQXCDvUUmweWsevf1Tr7nuT8yedWo1j6WNrTTnisJRUqtUWqwisgrooKqHneeRWEZr4mnAd1pNjC+cjBXnd8X4vXexvov1MBm45VjBfhJW/jJMVSdJEb0eC3gfD6av2gnjDd9R1RdFZDTGV04UkT7AC1i942ygLVYHORELm/bBOO+nMI3Wr7AWV/8CIrGQ7AJVvayoz+xykJUDm1fH9fHHH+e776fS4y8v8sHUZdyYnMXeLWv4/bdV/P7772RlZVGjRg2efPLJUuPNKsoelQa2gmoeN23aiFD+2CoCB1mrWRvt9dgJHQALtA9v6RD4HCSm9OK7LtiZq6yWgCW8DMAOyOuBP2Ctph4EVgA/qtUpxgC/iMjJen0pQGPNEzqP8X1RRMKBN4HzVHWdmJD5EbVEn2FAK0xFJwor/TgA3IKFXWOxg/Nn4JGC3lxcDrLSYfPquNarV4+09VtZM/RmUOXNRY0gO5PeZzdnx44dHDyY14o0NjaWb7755rTwkoGwR6WBLTk5ucxqHgN530qLgxQqSR0k8CLmwXyK6a++hXWoWA9MLO/YcFkMjMtL83n+HnCD8zgeqz+cj/F9i53xO9ZR41b81ELFZODWYp7oJUCQMz8a6EfRfSCHYeIB3tdSsdD3/wGP+8w/AdxT3Gd2OcjKj+3MM89UCfJoVO36GhlTWxHRxq07lCkvWdH2yMV2ejjIWs3a6A1jF/s1CAAOsigP0pupuhyTNvPa3KKP3ApvGT6Pc3ye52DeczZwtaqu8r1IRM7keO3T8Pw3luNDuDlAMlaWMUZE9mM1kdWxHydnishczNMToIfkeZoZIpKGeba1seSp7UBNEZkM1HXGe/5+aJGKw4GU91xFw9a8eXN2awT9n3yb9Ut+5ttXh5JwVndSp33CgQMHGDp0KF999RXdunVj7NixVXKPXGz+4yiplcEty8zcfpA+Jj41i87z0eRxgnGYV/gF1ndxkKqqiHRQ1UUi8gesxvEPWEH/cuBy9eEgnXusAS4AlgA/Yh55PDAU8yi92ai3AvcD/3Tu1RXLcj0Dq29so6oXich2zLP/BOs92RVT11kC7FLVzkV9ZpeDrBrYtmoUlw5+ivVLfub7N58mtu2ZhO9JLzNesiLukYut7DnI2vFt9dInx/u1duwNyYHPQYpIcywRpA0+XpGqtixDXIFsTwAvAUvEFIbWYYX7s53HK7Cw58JCrl8H7Mc6fMRhEnJ/Jc97bA3cDfwCjMAOzwWYtNx1mPeZgokD+Npq7Pv5xXm+G+MoTzBxOcgqh23/zq389y+90Jwc6jdvw95tG2lVL4Z7772X7t27s3v3bpYsWUJsbGyujuuePXtYtWoVTz31VJXYo/LGEcjYyomDLPduHsXGYLEO972ApUBz4EngifKODVfEgX+1jIuwQ7G6M78B8xDvxbzPutghW9t5fRhwH+bVbjlZTC4HWTWxeXnJXpddoV3Pu0CXLl2ay0t67emnn9Z3331X69WrVyX3qCpjKysOsnazNnrb+CV+DQKAg/TnZ0Gkqk5xDtO1qvowpgfq2imaiMSJSCoWDn1RRL7DNFVrYvWms4HZIrIW8zB7YB58JBY+3QaMcO5xJYCq7gd2i8gqEVkoIh+LyNkngQnnPqgGNgdS3nOVAVvz5s2JSzmHM+96lrP+dDee4BASzupOWloaOTk5uTqjoaGhtGzZskruUVXGVhSOkphI5auDnIPxap9iCSabsPq+SlcHWRYmIj9jhx9YI+Nm2KH2NfANlkhzLdby6g6gG6Z5ew6mtXoFJk0XDjyHlZ4cBTpj3ORG4EOsFdnXWPi2HtAA81YvLAqfy0FWXWxeXvJQ+jL+2qMdN973GI/feT0NGjRg5MiRxMbGkpiYyDfffMOsWbOq5B5VVWxlxUHWbd5Wrxw+wa+1b13XLvA5SCy0Vw3rQfgUJqg9oCxBVSZT1S7558SSddap6uXO8+YY//hXrHFyDHY4TlHVJc6vuC/VRMubO/NLnWunAjOBs7EQ+EZMfGAXViN5gonLQbrYyOMlRXMI23gzYDqjF1xwAQcPHqRu3bq89tprBAUFMXfuXK677jpmzZpFVlYWycnJVWKPynsu0HCUhokEhnfol5V3jLcqDk7kIu/DuMR1QLIzdysw2nk8mnxcpc+1o7Hayb7A+HzvIzg1loUNl4N0sWVkZGibNm100KBBKmHRGhzfW0Na/UkRj44bN0579uypQ4YM0eHDh+vTTz+tnTt3LvY9KuMeVSVsZcVB1olvowM/XubXIAA4yEI9SBH5DKcHZCEH6x9P5UCu6ibmPU4FaonIcixk/SPGPzYG3hORA9jhlibmMV4KXCDWFuteYJ2IVMe8zRSsbORhoJuIdAdGYpmvZ2FdWNb7gQuoGBxIec9VRmy9e/dmw4YNSFgMDZt3YtvWTeAJY926daSlpZGWlsa0adO4/PLLuf3225k3b16V26OqhK0oHCUxIXD4RX+sUA5SRIrkrlT1hzJBVMnNOSDXAOmqmiAiH2Ftqs7GvMHbsPBqDeAHLEknCquRrAM8j3maKVjSzitY/eNDWNj0Raz7ylrg76r6RVF4XA6yamPzarh+++23jB8/nvU7jnHp9YO4tEss/a+5nH5X/5E+ffrQt29f4uLiGDBgAC1btuT666+nWbNmVWKPqiK2suIg6yUk6TUjPvZr7X/+2CZwOUj3ACxTW6eqLZzHCzDR86uAP2LNjgEOquqtIrILqK+qWSJSA0uQShKRECwxpyOOtitWg3kFME1VEwp7c3E5SBebM+fVcO3YsSMAq9f8xqqHbud5zYIsE5HyXe/xeBARsrKyCAsLw+PxsGDBAhITEyvtHrnYSpeDLL07nQYr7xhvRR7k4wP9WD8ak4XLzz++QCE1jFiyTbDzuAZ2cIJxlBOwzh4TsYScuJPF5HKQLraC5latWqXBwcFard1l6qnRQD1R9bThzaM0pHacvvbaayoiumPHDlW1esn8Oq5VYY8qM7ay4iDrNW+rgz5L9WsQABxkhTrMK7Htx3jFawDELNl5bS5wtfP4Tz7XRAPbVfV34HVMgu6UTaTicCDlPVdVsKkqOYf3EBLdgMiErhxOm0lki3OZNGkSAOvXr8+tl8zfX7Kq7FFlxVYUjpJakPg3AsH81mIVkTBVzSh+ZemaiPydvLKSt1T1pULWPYS1fNqOqc8sUNXnRGQ6cJ9aiUQd7FdJnFhPxn8D52N1iq+r6ptF4BiC1SuGAZ+p6lAxPvFbYBZW9L8JuEJVjxRyj9FYG6q/AR9hmadNgM1YJuobWM1pBnAM8x6HYpqsccBOrB3X78B/HDw1sT6RW7FSnBFAIpCGiaqnFfaZwOUgXWwnzs2ZM4cjR46Qk5ODiofQ+i2p0+cBdk0ZQeau9TSqHUVGRgb79+9HVenduzc33njjcTqulX2PKju2suIg6yck6Q0vTPRr7YtXtA5cDtJrItIZa+4bDTR1PJs/q+qgsgYn1iHjNqALWI9DEZmhqosKWPcnjIsLxnRQFxRz+9uBfap6loiEYco136nqugJw9ARaYMX5AnwhIudhB1ULoL+q/kUs4eZq4P0i3neHGodYS1Ufd+4/FkhS1UucAz3Nud952KGZhB2UtwHfOX8XOfsSiiOwLiKvAiNUdZyIhAKeggCIy0G62IqYa9SoUS4v+eXPa1CU7Z/8EzSHsMZJXNErhbPPPpuIiAi6d+/Ozp07+eijj47Tcd28eTNjx47lnXfeCaj9qArfX1niKKmJgCdQ3EN/rLgYLBbiOwNY5DPnN8dVkoGfPQ6BwfnWvYB5jQDTgU7O4zpY9igYb7eavL6O64CeGLfXKN/9n8M4Pu/aNdgBG8fx/SPfwGTgFgMRBeAcTV4949WYN7kU8zz/5YO3h88125w127ByjQRn/j1MkSfO+31gzZ2XYx1AWvizxy4H6WI72bnEnv21er0m+tBHc/ShL5frsuUrtH///sfpuA4cOFBjYmLKfT/c7y+wOMj6CW11yFcr/RoEAAfpj5JOkKquFznu1M/247pAsSzyEqd8ezQK1rJqiu9iMQ9uGRb29F07XPOFYMVCrL5h5xRgthZTIyoi4ViItJOqbhCRYfmw+ca9M7Cw671YSHxNIetQ1Q/EpO0uBb4RkTtU9ceisPhg8t4D1cDmQMp7rqpjO7x7G9XrNmJDZjWiNQORCEJCQnJ1XLOyspg6dSo5OTk0adKkzHCc6lxV//5KiqMkJlA56iBzF4h8AjwD/BcrPB8EdFPVa8ocnEhHzOs6GyfECtykJ4ZYveu6kBdifVONg3wLk25rhTUXDsNKI2ZjuqbXYJ7qGZi3/Cbm0R0BzlHVI2Ih1ieAC1X1oIhcCzyClWfUx3RPb8JqEjOAb1T1hgI+zwSgHfYDozXWJeVn7EAOw/jTplh26oPA59ie/4qFVM/BelK2wLzHTZgowI1qIdbhmDBAGHAQ+EgL4Wy95nKQLraTnftp+Tr2blhDtdoNaNKwPh+PG8OrLz5Pnz59cnVc9+3bR2ZmJqNHjyY5ObnK7VFFx1ZWHGTDFkl628uf+rV2+KWJgc9BYmUJr2D/cG/DVGDuLEtQXlPVhWKJLd4eh2/lPxx91k3ADpLt2MHntW8xTnA1Fpa8F0ts2YPVDS7EQq9HMTWa+ThJPT73/05EWgP/E/tJ1QLogxXjLwLuVNWXROQOrMbxhMPRsXhgsapeLyJPYW2sdmAtrEap6mARmeVgWQXkAN0xndYPMP7xFkyM/F4s0WcJsN85xHtg4gGZWDJPgQk64nKQLrYSzAWHhlM7PolDOzfx+/p03n77bcDqJfv37092djZdunRh48aNdOnShYiICDZt2kT16tVz6ygr+x5VNmylWQcp/juQgd8PsiIOnB6JzuMCeUwK0EJ1Hk/H4SwLuXcyMNPn+YXAp87j0Tgco/O8E/CKz/PzMP5yGJDizPUFxvmsme5ctwiI95nfh6nlDMv3ebxcZH6edD/wn+L2yuUgXWwlnWvTpo0G1U1RCYvW0Ha3aVjHe5TQKA0LC9PDhw+rqmpSUpJ27969yu5RRcNWVhxkwxZJ+vC3q/0aBAAHWezPAhEZJSIj8w9/Dt8AthiOF3QIL2xhSUxV56vqPT7PZ2KH5CZgtIjcfKq3LuC5lydNUdUUrD2ZX/wjVCwOpLznXGzHz/Xu3RuyDiFhMdRv0AQJ8uCp1ZJjx46xcuVKDh8+zPr16znrrLPKFIe/c+73VzIcJTVPkH8jEMwfDvI6n6fhmCTaBi2lMg+xRJfJWFlGRywL82aMo3sBCy/uxLJLj2Fh1DrYoXAUy+x8BPMMcebPwzi4Zx28jbGQ4qdY+PWg816rnWuHOBg+wWoadzhrvP0bJ2B1jvOwcO3nwBasDvEurF6xK9DSwTofaOtgr4tlyE7GvMVMB+P7QCzWQmwhcJ6qrhORWsBfHEyZwBjn2v8Cex08YRinmkleQlFN7OBvr6obxUpOJqvqO0Xtv8tButhOdc5Xx/XdMWPZti+HS68fxB39uvH5px8z+q3/kJqaioiQlJTEI488clytZFXYo4qKraw4yMYt2+kdr3/m19qhPVsEPgepqsd1txSr2ZtVyjgSgdtVdbaIvIMV0l+FFd3vcA7pp1R1gIgEAdGqmiEiMaq6V0S+BP7mXF8dOzj/iGWVJmKhybedvyOBt7AD917soJrgfNalIjIA0zjFwbAcEwgfQF5STx+MP/weO1SHY4fXW9iBnaSqKSJyPhbqvczBWB071A5hYdGbnc83EPjU+WzZDv5kTCGnLyZm0BwrDemIHZ6LsEM8GxMlnyEik4HFIuLN3J1R0GaLy0G62EphzlfH9acFlly96qHbeeGhHCS8NjWD9xEfHw9AZmYmM2fOZN++fYSEhBAaGhowvGSgfFeBjK2cOMjyt5ONyWL/UK8trRgvVsf3u8/zHlgi0H7y+LSlwHfO65OxGsYbgerO3L+wbNB7gCbO3IvAAJ/7jgUuxzJPX8OSWxZj2aoNisDmW+f4HnCD89ibcAPmCX6GeXNLgZXO/PlYEb8X435M6aZJEfvxPPCXfHPRWLlKHYyDHI55ndH59q65M3+QfHxoYcPlIF1sZTEXHR2tBAVrq3ve03YPf6shNepqRGxbDQ8PL3deMlD2KJCxlRUH2ahlkj45Nc2vQQXhIPeIyG5n7MW8pgeKu+4kLX+c9wCwXB0+TVXbqWpP57VLMc+qIzBPRIJV9d/An4EITBGnVRHvdQMW9jxTjavbRtEcpG+dY47P8xzyPPAnsA4aSZjHV11EVmIea3cRmYhlAq/CwqOzRWS8iMwXkeUi8hiAiPTAunHgPL9YrC9nfvuH87c+UE9EFovIMiy5p5GzD5djPxj8MpGKw4GU95yLrfi5nJwcQAmpXgsJCsYTGU2QJ5TMzMxy5yUDZY8CGVtROEpq4ud/gWBFcpBiuxOLJZUA5GhRF5wKAOMg1wFdVfV/YnWLaRgPd5MzF4Lxe6lAU1VNd+bWA22A2qq61rnfRCxsGgTcgYVDa2G8YBfgOkyNZpCIXIAlsjRT1fQCsE3B+MxVzlRTTLP1EQe3V+LtM+B9Vf1ErOj/r9jhtQnr5ZiFhU3TsIzZfwGfq+r7YpqwP2CH2VJMvm4L5n2+BXypquNF5BjwR1X9ynn8OrDRud+1wBzsoI5w9m6ig69I4UOXg3SxlRWO+UtXEhxRAwnyEBRejfB6zeh3ZiNGjRqFSPnxkoGyR4GMraw4yCaJ7fSe/37u19p/9kgIbA5SVVVEvnE8o7K0VcDfxPjHFVhz4CnAKyIS7eB8CUuqed+ZE6yEYq+IPOEcdjkYZ/gtltBzDpbUo8D9qrpVRMYBX4rIUuzQXFkErjuwQyYFQKwm89cC1j0LjBGRh4GvsQNxA5Zs5G1nNQor+h/vvG+MiCx0PltDoI2qLhGR/zq4FwLNsB8B47Hkn2Ei8jR2+D+OiQ5kOq8fdvbuNuyALNTE5SBdbKdhLig4FFByso4RFnUGmpPN3Llz6datGyNGjKBu3bpERkaSmZlJ//79AdiyZQs5OTl89dVXvPDCC5V+jyoKttLlIAPDO/TLiovBYt5Yh7KK8XKS/QsrwnA+03qf5z0wjnI6FgZthtVD1nRenwhsdB5fjHmadwLP+twjHajjPD7oM98IOxAXY0k/x71e3HA5SBfb6cLRu3dvDQqN0HMf/livfX22htSoq/WTuumGDRvUaz/++KMOHDhQ69atW+n3KJCxlRUH2aRlkj43fa1fg0DmIEXE6112wLi+VSKyUEQWiXk+5WpioclAtqYico7z+HqOz/ytgWWy7hOR+phajtcaY17lw8C7Rb2BiJyBcahvYVm6HZ2XMsVC0H6b91ed93+Mgn7lnc65QMHhYis9HMeOHUNV2ZsTwcpth/FERnP4aBapqals2bIFVeXJJ5+kW7duZGRklJmOa6DsUSBjKwpHiUxA/ByBYIVykCKyUFU7ikjzgl5Xh/MrE1DG7xVUG7kCK8m4GAtrrsTqAyMx2bcBqrpHTHB8EXAuUM259gEsJDlBVR923sfba9KDHVo7sHBnJhbSvBDTnj3ovM8krN4wBHhYVScVgX8OxgdGYIdYa+A34AVVfVZEPscScg5iIeBwzAtsgNUzehxsrZ01d2Pe53TMA20L/M+5LhzjL1dj9ZFxzj2/04I1YX1DrGd27NixQnAgLk9UcbAVhmPGr2vRzIxcXrJFchf+99FrdO3alRkzZvDqq6+SmJjIqFGj+Pjjj+nQoUOl3aNAxlZWHGRsq3b6j1Ff+LX23vPiA5qDFCjbg7AYy18beZczv0tVOwKIyBKsI8cMEXkc65k42Fl3TFU7icj/YQfbmcBuYK2IvIgdIrfh02vSeQ7wkqp2d97jWkxU/ChwlaruF2u8PFdEvtCCf2G0wITRG6vqTrHej4fF6hR/A1DVK0XkoKpGSV7CT2cRuRX4O/Cyqk4QS/pBVeMcPGCHI1giUFdVneu8VktVd0te4s8zBW2smp7hSLAknYrEgZQ3DhdbyXAEV4shc+9WcrKOUb1mRR8ZCwAAIABJREFUPGOefZAZM2bw8MMPk5mZycCBA5kxYwaxsbHUqFGDtWvXsnv3bjIzM5kyZQpPPfVUpd2jQMZWWhykABWpHWShsVcsQ/LvhY2yjPtScG3k5xgPd4YzV2ANoPN4OtZxxHvt9z7rZmICAgVptD6M8YGpGLeXjLWvgpOrnxwKbC9gfjTHa7UedP6+jImc43zGTUCY83wYluhzq4NpmbM/cd5rfO73Vyy5ZwnmDf+puL12OUgXW3nhaNWqlY4cOVIltIb2uPEJTUvfosuWp2pI04uP6y359NNP6+TJk0uNlwyUPQpkbGXFQcYmttPXZq3zaxDIHCQW4quOlSkUNMraCtIbBePu/DHfesX8tYyFec77VbUf8DHQDysJ8SoJnUz95G4sazXXxLy63N6UYqo5oc7LL5L3uYYBn6mqF7P3mluxA9L3PXP3QkSaYQfpharaHsum9VtjVqTicCDlPediKzkOj8dDu3btmDp1KhIWzZxlW5kxbw2R1aqTc2w/aWlprFu3jmPHjvHhhx8SERFBVlZWpdqjQMZWFI6SmFDJOMjTjMf73nGcWBuZivGBnVR1p7PuV+BuVf1JRH4gj69sgZU+pDh/+2IHzHzMQ3seeAzj91ZihfevAPdj3OZ1WPZuK4zbzMHCtDGar37SgZyfLx3uvG+48/cSZ+4PmHzdLqwso4uqiljtZmdVbSoiQxw8mzAe9EvgT85n2Yn1rWyFhYzfxrJh52ENmN/GSlyuwpR+pqpq7wL21+UgXWzlhqNWrVqMHz+e1157ja+++orIqJrkNDiXLTv2M+mV2xky5F/0v+5qBg8eTHZ2NkOHDiUyMrLU6iUDZY8CGVtZcZBntG6vD7zjHwd5Z9dm5c5BFuVBlvcZ7q2NTMUSY94oYM0twAgRScMSWNpiwgA1irn3bkxc/CEsiedrLBN0BYCqLscEEtap1YB2whopdxKrn7yZ4+snE7HWUq0xObkLMBFybyPlRar6IRbu3Yt5w9lYMlCuiWWeXoMJBXiwHwVnYYepYDWY3vd9DhONb4d5xH9w8P8dC49/6YwTTFVHqmonVe1Uo0aNCsWBlDcOF1vJcSxcuJCEhARmzpwJwKTJs/li1EMs+vJZnnlmBOO/mUdQUFDutZs3bwbI1XGNcPpLLlmyhJkzZxIaGsratWu54oorKvQeBTK20qyDDBLxawSEFRZ7BWqVV9yXk6yNxBJzXgAmOs9fwMKN03F6O2I6punO42hMm3U5VtB/OP/7YqUZy4F/Ai2KwXoCX+o8Tud4znSTz/19OdPRWEg3icI1aKcDs7EM1wJ7UmIH5a/AO5hYe2hxe+dykC62QMLRpk0bHTRokEp4TQ3tNEgjz3tY27RN0trtrzxBxzUlJUXPPvtsbdu2raoaVzlw4MAKs0eBjK2sOMgzWrXTkXPT/RoEMgepqruLPFkDzw6q8Ye+lsv5cTwfdy/GIV6GNSIOzXcdqvoBpmd6BPhGTCe1MCuMLwX/OVMwL7EwDVqA/1PVvYWCUM0COmOJRpdhoV//3lgqDgdS3nMutrLD0bt3b7Zt24aE1yQooiYXntWSP1zQmz27tp+g4zpw4EA2btxIZmZmLlfZrVu3MsN2OuYCBVtROEpqFcmDLLYf5GkBIfIQFi7djnl0C7B/4O9T1fliZRXzVTVOjJ8ci4VGweoDjwLjMK7wbCzTU5w1R7CQ56XAYOceL2JhyLMwTysEC1m+Dsx1MERhNZNDReQ5LFzaHjv8lqjqTSIyHuiJab1mYh5iDUwWrwaWKbsGeEytZGMFEK6q8SLyJMYV5mDJN69gPGUq5nk2xOoa47F6yKEO3iSsLnI3VjIi2CE/ARMWGOmsESBOVb375LvfLgfpYgsoHL69JcePH8/6XdmkR3Rl5ANX8t57Y5k2Yxa39+1wnI7rr7/+yrvvvstf//pXmjRpwoABA3JrKCvCHgUytrLiIJu1bq9D3/vKr7W3dT6j3DnIYvtBlrWJyJnkJaEEY2UKC4q4ZDtwsaoeFZEWwHi1esevMMHvb7EDcz5WuvGxM3y/FQ+WCLMXO1wiVHWIiNyEeZOdMUHxB0XkeuyQWgecrU5do3OfDOxAbY7Jw/XCeL812KG2GevhOEVEZmJh34li9Zu1sZBvNzGx87uxusVJQH/ncyYB3kSg0ViyzzQH3/XYodnYWXM1puHaG9NvFef9TjB16yBdbH7OHTlyhLS0NGJiYggJCSE8PJyDBw+WOg7f3pIAq9esQzMWccsfX0WqNwRPGHPnzmXChAn06NGD3bt38/7773PBBRdw2WWX8cknnwCwefNmxowZ435/ZYSjxCYFe64Ba+Ud48X4Q996RH/5w6UYT1cQf/gJsJVC+EPyagtzr3Hmn8O8Ny8HuAa4HTsInyrgPqMxL9H7vgecv4X1ovTF+BnQw3n8JVa72B6r97zA59qFPvuQTp4e6zCMb9yKeclnY8lMazGx90uAoOL23+UgXWxFzfXt21fPPvtsVVXdv3+/1q5du1xweHVcb3x5sl77+mxdtnyFXnPtdboybZ0uXbpUj2aqHslUXblypdarV8/9/gKUg4xr3U7HzPvdr+H8ezffZww83edTmXqQ4l/o9FHgNbE6wX9jnlEO5tkFicj5WOixnoj8hnll+5xDIYIC+EPyZNeCgEUishHrMTlYVWc7a9oAHwItReQeVX0F87qGq+qb+T7Hf5y/1Tlebm47psATItb/MVws69aD0yJLRNIxHvJczGP14m0FXCsiN2KtvGIx7jDSuW4YFkr1YliG4/GLlb3cgMnpeT3KcKwEZBdW1nI1VopyYwH7c4J5f9V5/8co6Ffe6ZwLFBxVHVtcXBzLli1j3bp1NG7cmMzMTJo1a3bacXh1XH/dlkXjOjkQ5OHQIaP3zzjjjFzSf/v27bRs2dL9/soQR0lM4GT4xZ0awGUeJTI5PnTaB+PPCrKjwJWYCswR7CB7GfMavQkqKdg//K2dxyGq2hnzugoSLX8PeBDTXv0S01G9Givl8ForjAPcBgwVK7GYAgxwDkJEpLGI1MNKMq7DtE/rYocjWM/GI1hoNRHIVCv12ADc6xz6QRjHeLaDq4Fz7e9Yl5QBWGbqDgfvm1h/R4B6zpyvJQMXYQfjJVi5SlPntXHAUDVZukkOrhNMRAaKNWuev3v3blq3bk1iYiLVqlWjevXq1KxZM/d5nTp1OHDgwHFrTsdcoOCo6thycnK49dZbc+cyMjJISko67TiCg4MJrl6TZf++kh8euJi3Ro1k5cpUzunckYYNGxIZGkS9WtFs27aNFi1acNFFFxETE8OBAweYNm0aLVq0qJLfX2nioJTOC/FzBIKV2QGJeUyfqephVd0PFFYdegxLMPk3MAQ7GO/BwpoDMEWaLUC2mrrMCiBeTCQgCDu88tu1WKLONVhyzu3O+9fwHn7A16q6BfgJS+Z5RVW/Az4A/idW7zgRS9bJwfhMwQ67xs77Knk87gZnHVgNZB0sEagB8KiqbnUwRDpr5gHivE8EFk64Bjsg6wJ/w/jE5ZjH7LWzsYNQMO81Eztsq2HJQk+JyGKsQ4jvdbmmPnWQUVFRpKWlsXLlSg4dOsShQ4fYu3dv7vN9+/Zx4MCB49acjrlAwVHVse3cuZMxY8awYMECkpKSaNCgATt27CiXPco+tJfEu9+hx5Pf8u2339KxYydiatbkjTffYtbs/3HsWAYbNmwgKCiIq6++mkcffZQ33niDmjVrsnLlyir5/ZUmDgp2Rk7ShKAg/0YgWFkekIXZCaUXqvoU8B3mSb4MvKiqTVQ1Hut1uMLxisCK5v+kqsnYYTLZuUe6Oo2dVfXfqtoWOyDqqZVLpKhqY1X1hi0znLXXY1zjM87zl5317VT1HHXE2lV1DMZR/gDUV5Ob2whMde6nquo9fMHCr89jIWFvgtB6LMzq3YeP1Ar9a2KcYkdgBhaWfhkTMIh2rgvHxAAOYT8Wejve6heYiMIszIPt6GDri9VUFmnZ2dmEhoYSHx9PaGgo9evXR0Ryn7du3Zo9e/Yct+Z0zAUKjqqObenKNdRp2JS//G0w3fv0o881tzBr1qxy2aOaUZGkvnwTk+87n35/vIpPvp7B+p3ZDHzpFy64YyQZ2R5Wr17Nhg0bmDRpErfccgsbN24kISGBrKysKvn9lSYOSiGpU7B//P0ZgWBliWMmcKWIRIhIFPYPNpindKbz2LducQrWJDgIQERaisgJJQoFWIKIrBaRWSIyXkTuE5HpItIJO3T/6fCAiEhHERmBHbp/F5E7nHtUAz4UkUki8puI/FtEbhCRXxwPz/tzJhoTIc8Uk5s7A/Pa3sD6P64QkW4Yj/o7xnM2AOaLyD3khWlzTUyT9TVn7T+xcOsaLMv2rxhvuwLLlL3MZ18fEpHlmKfZR1X3AYdE5GcRWYDxj0uK2zyPx0NGRkau7uW2bdty072PHTtGamoqNWvWPG7N6ZgLFBxVHVuwJ5jNG9cT17wl199+N/UaNOLw4cPlvkcxMTEQFISERlG3pvO7VEx9Jy0tjS1btlC7dm0+/PBDevToQU5OTpX8/koTB/ajvsTmmy1b1AgEK7MDUlUXYqHTX7HSi3nOS88Bd4rIIiwM6TWv1NtVmG7qmxT/i6UlFvIsjOe8x3mtkYgcxbyzG7FQgQJ/FRFvGUUb7EBqjcnDtXR4znDyEmvGcaLc3FAsk3S18/xH7CBbivGcW53rXsAydu/Nh9GDhU2fwZJuXgb2OJ8lHeMZD2A6rA9g4eaFWOeRfZiXOM25134gDONI07HylBPMl4M8cOAALVu2zOUaqlWrRkxMTO7z6OhooqKijltzOuYCBUdVx5aVncXG9ev4dNw7/KFVPZ5/7H527txZbnt07rnnkpiYyM8//8x5Z7WmYXQQBxe/TZ3d31E9zMoRbr31Vl566SX27t3LpEmTaNasGcHBwfTq1Yvhw4ezYMEC3njjDc4555xK//2VJg4KprNO2lwO0jFVfUpVW6rqH7ADBFVdqartVbWDqj7sDZ2qao6qPuiENpNU9QJV3aeq01X1Mp97nq+q852nkVjvxgJ5TjVR878Am1U1HPNSD2CH1g7MW2uB9YH8WVW3ODznWsz7BAtpzvbezwm7tlPV25wQ5zlYRmlT7EDaCSRgGahfO/d4HKuTvFJV1zj3Gqaqz6lqpqp2wDjMntiPiT3OumnYIT8JO0B/xA7OXcA9qtpVVeuq6hMOt9reeb8czNutW8j3kstB1qpVi9TU1EK5h507dxIVFXXcmtMxFyg4qjq2Bo1jCQsPZ8J3vzBtyUbCwiOIj48vtz2aOXMmq1atYtasWcxcuJ6Nv63gUINe7Ii5kIMHDxIUFMSYMWOYMmUK2dnZZGVlsW3bNkJDQ1m9ejVDhw5lxIgR3HXXXcyZM6fSf3+liYO8HItTN3E9yECwwiTmBGuwnILxnccwj/JdoK6I/E9EFmKen+/eeABE5A3H81ouIo85r0X63HuPqjbGEoQiMZ6zBpYI1AxHvUZE4kQkVURGOff6DquL7AeMAJqLiQn08cHQBrgYK/DvkPuBRLyc6nkO5k1Y0s9c8horF2sVKc28vOeqErZakcEEiZByRg3Oio8mxCN+XXs69ujqi1OICA9h5aRHWPbpg3g8Qbz1zUI2Z9bg27Rj9PrbCI5JOJMmTSIkJISDBw+SmZnJ5MmTc8XQywrbqc4Fyv9bZVrm4ecIBAsIqblTNRHpiBXrf4KVcSRiHlw4MEtVrxCRweRJzA3EDp1rsEzU3zBx8WjskNyI/UpqjXmaG7HDca+qnisitVR1t1j5xg+Yd/cvLAzaUk1lJwUr9v8IC3VehIVQ52Eh4+uwEOoaTABgsYh8hHGNV2A1jc1Vdb1YjeRN2GHXG6jv/J2LZbdeiCUDNcW8xx+xlmD/xZKXJqvq80XtYfPmzTUhIaFCSF25cmAVB9vpxDFnzhyOHDlCTk4OKh5C67ekTp8HqL58HJ+89Ty9L+nFc889R/v27fn+++8JCwvjp59+4siRIyxevNj9/k6j1FxC22R9bvwUv9ZeldzQlZorianqQhGZQJ4YwScYP/cV8JHDc37tc8lbmJrNQoxXzMB4v7uw5tDeEEIWMNs5YBf7XH+tc8gGY1qpbbBDcj0wQ+zn1kyf9c2wUpdDIpKDHVrnYqHgY+Ttf3ssBB2F8YrPi8jn2EGbjCUvRWLZvStEZBMWfp2PeYv3YSHheVgy1EAsvFody6Q9zuR4LdbjwhqBLnVV3jhcbIGHo1GjRrkydV/+vAZF2f7JP9kl8M6Y97niiiuIjo7myiuvJDs7my5dupCRkcHatWuJiopi3rx5bNq0iYSEhCq1b6eCozRMAiN66p95XenKMHAk5PxcG0ee7FtfTNO1oHXTsX6QzTCvr6YzPxq41XmcjiMB5/sck6HzldF7AjtQ4zBu0yshdx8wzHnswcTVX8CEy4Pzfy4shBvnPD7o/D0fmOGzZgB2oBa5D67UnIutMuEIj++m4gnVhje9qV0e+lpbtmqtgwYN0vj4+Nx1CQkJ2rVrV/Xak08+qeHh4RoeHl5l9+10Sc0ltGmvXyzZ6tcgANpdVWgPshRtLvC6iCSo6hqx8pLGqrraZ00N7FDbJyLeUOd057UDmPe3U0TisPDt69jBGioivbDs0mbOdfntQeAtsZKPZ7Es2J1YGcm/nDXXiUi6qk7EafIs5pWGiUiYs+ZcEXkZ81JbYA2h/TKRisOBlPeciy1wcVzWsSFfb/Ew+7GLady4MQ8e7cOL730H2R5e/CKNhrW3sC4nnl2p83Ll895++22aNGnCpk2bquy+nQyOklpF8iArNAdZEnMOsq/UERcQ6/f4DHaQATysql+IyHTMw+uF1SZGYuo12cBUVe0nIoOwbhybsYzYdRgvuQBTuzmKlZVsxEKmezC+dA0mVXcGphj0KSY8kI7x1PUwhaHqmCLQM1j4+ADQU1WniUgmpkC0GAvhriFPJ3aVqv65qH1wOUgXW2XE4ds+690xY9l2yMOmmhfx5n19SF8ynTnTJrNq1Sqys7OJiYkhJSWFjRs3ut9fGXOQLdqm6EsTvit+IXBZu/ouB1lepqrpWDsp73NvCUX+dec7D+djEnIF3etVrIOG9+BNV9UpInIZlgB0wFnaFPhGVW/3Hrxqou3pmOd4I/C8qg517vWCc/9hzn3BDtb/qaq39vESTCxgMcah9sIOyB9w1ILym7gcpIutkuPwbZ/1068bAEWPPssNM59BohrRp/MZiAjbt2/nt99+Y+vWrfTu3ZspU6awe/duzjvvPNatW0d4eDi1atWqMvtWHI7SMKlAHmS5xncr4+AkuU3ncTrGWQ7Gmit717yAwz1inGc/zAOd6bPmQszzPB+T4avjYFgDTC8Or8tButgqM44GDRqo1E1SgkI0tOsQDbvgSSWkmkZHR+vatWv1scce04iICG3SpIn27NlTVVWHDBmivXr10oEDB2pISIgOHDiwyu1bWXGQLdom6+Tl2/0aBAAHGSjlJpXV5gLdRCQBQESqiUjLItbPBvqKSLhY4X9frLvIr1gJSFesnKWLmLzeSKwMZAbmpQZjB+/fThaoSMXhQMp7zsVWcXAAXN29FRFhwax8/072Tx5Cragwjkg4N3+6iVc/nUFo0/Zs23uItLQ01q1bx+eff87vv/9Ot27d8Hg8fP/996cNb6DsW1E4Smoi/o1AsCrLQZaVyUlym5oXYu2kVkc5DNNy3YYdeJmqep6IjMbqHL/ChAKewzzF+VhSzzystKMdpul6FbBR80LEBZrLQbrYKjOOwrAtWbeV5P8byfynrqHhudeyZeYEJk38kMGDB7N27Voef/xxEhMTGTVqFFOnTiUuLq5K7VtZcZAtk1L09Y+nFr8Q6NmmbrlzkK4HWcqmPl1FnOc/qupZavJ67VX1C2f+fHUk81Q1Tk0WD+A5VW2JcYnVMTH2Z4C3VfU9rATkaezwPIxxjRFAjKo2dO4zFjha2OEoPlqs+/fvr1AcSHnjcLFVbBzeueyMIyx+6c8ER0RxYP0yAIKCgti2bRs5OTm8+uqrAMTGxlK9enXi4+OZNWsW8+fPJysrq8rtW2lxkAIEiX8jIKy8Y7xVYeDwh87jt4A2Raz9AEu4WYmJk9fCkndmAI9inmWss3aYM2KA333u0R447A82l4N0sVVmHIVhI8ijZ/7rQz1n+A8aXC1GPRFRGh8fr+PHj9fY2FgNCwvTcePGaffu3bVmzZo6fPhwffrpp7Vz585l+rkCZd/KioNs2TZZf0jd6dcgADjIKpvFWl6mxZRdqPWnBEBEGmEH3fsishfwXrtTjKPsB0xU1b0isldE/qCqszDx9N9OBpdIxeFAynvOxVZxcBSGzSPw8cDONG7cmCYvhHD0aDabM2tw/+xIdke14tjWHdz5yjd4Vi8hJyeH/v37c/nll3P77bczb968Sr9vReEoqQWVwT3LylwO8hRNTEzgI6AJpn7zBFaC0RcLec4B7lBVFeMPv1LViXI893gJFi71ADtV9cJ879ELEy8Px9p6bcREAhTr/pGD8Y5bsU4ioc7j74B/qGqxHcBdDtLFVplxFIdt8+bNZGVlkZGRgQSHEXPeXwipE8/2ifcjwaG0b92CpUuXEhcXx4ABA2jZsiXXX389zZo1q9T7VlYcZKukFB356Y9+re2eWNvlICuwXYK10UpW4xwnA6+p8Y1J2CF5WWEXi0hdTBzgalVNxgTUjzNVnaKq7TFFnhhVbYUJEcxT1W7A21idpa9Q+59V9X5MgKCw93Y5yEqMbf/+/ezdu5eIiAiuuOKKgMAWaHvknWvQoAGXX345vXr1IrRRaw4s/JRd3w5HPKFUSzyff//737lrPR4Pu3btIjs7m+DgYMLDwwkPD68S+1Z6dZDi938BYeUd462oA2vWnI5lqJ7rzF2NKecsxdpO/cuZH00eB7kVuB/zNNdQBB/p816xWDusZc69VzrztwLv+ax7HOtcAqb006m4e7scZOXCVrt2ba1Xr54uWLBA27Ztq08//bQOHDjQ3SM/sHk1XBvc8B9FgrRau8s0Pj5e4+LiND09Xdu3b69PPfWUhoeHq6rq/v37tXbt2pVy38qKg0xMStGfVu/2axAAHKTrQZ6iqem0dsQOrCdF5FHgP9hB2A7zDsOLuAVYk+YVfrzdE8A0Nc+0b7775o+Rn1LMXKTicCDlPRfI2IKDgzl69CiHDh1CVfnwww/p1q3baccRyHtUGLbwUNNwnfHohXiChEtbR7I5swa7aqaQ3P8R1kcm8/h/PyIsLIx169YRFhZGZmYmzZo1K3NspXX/0sBRUhM/RyCYy0Geookl0OxW1aMicjUmTl4ba1v1DMYdZmG6qyHAQ2oc5FZMIeddjEe8Vq0ush8mWn4CHykinwHvq+onYhxmJ6xNV2ssTP4+0APTdL1IVX8WkWygizqlJIWZy0FWLmwdOnQgIyODZcuWsWnTJoYOHZpbz+fuUfHYfHtLhoaGorWaU6fPA+yaMoLMXeshKJgP3n6dRx55hKNHj7J//35eeeUVxo0bV6n2raw4yNbtOug7n00rfiHQtUXNcucg3SzWU7d2wAixjhrVgf9hTYz7Y/0lpwJpajqqa4EzgYnei1V1h4isBp4VkaeBBKC1qq4TkVr53suD9Yh8mDzFnB5Yp4+hQHdMiGAj1qeySBNXi7XKYPN4PAGBI1D2ozhsvr0l9+/fz/TUbWz/5J+gOYQ1TsITEU10dDSqyvbt2+nXrx+RkZHs27ePkJAQQkND2bRpE9WrV8fj8VSafSs9DhIkUNxDf6y8Y7yVYXDqfOR0zBvsC4zz872GYd4oWMJONnmRAF8OcjouB1nlsNWvX19r1IjW73+Yoa1bt9Z27drr26PHuntUStgGDRqkjWLjNLnTOXr3P4dpQqu2et+jz2p4eLgePnxYVVWTkpK0e/fuFXrfyoqDbJWUonPX7PVrEAAcpOtBloKp6moR6Qj0wfjIHzA91E6qukFMPq44PhIAKbh85E6c0hBMZq6riFyBKemgmltK0hqoLyL3UEQWayHv6/0sqAY2B1Lec4GMLTMzk+iYaGKbNkWCgrjmuj8xe9ZPpx1HIO9RSbBt27aNYxlHaZWUzE0DB5OdncPWLRvJzMxk5cqVJCYmsn79ei655BJ++eWXCrtvReEoqVUkD9LlIEvB5Hg+8jKsoP8cTCvVg4mWT1QLt44mX00ksB5YCLyChWejgd+d28/Dmh/fh/WbXAE8j/WJ/BHjGUOd+56FeahfY70oz1WXg6xy2FakplItMpKdO3dSPSqKRg0b0aRJY3ePSgHbxo0b+fnnn0lIbIMEBbF/3x4SEpNo2+IMRo0ahYiQlJTEI488ckq8b6DsW1lykO9Nmu7X2s7NY9ZjpWteG6mqI0vy/idrrgdZOubLR2ZiHt+VWFnGVuyQK9TU+MiBmGjAUaABMAU7SH9yDlKAzlgo94iqZorIx0AXn1uttNvpChEJKez9xOUgKzW26BrRRERGEFmtGh06GJcmUr54A22PThVbTEwMZ5/bg80bfycnO5uklLOoVr0Gc+fOZcKECfTo0YPdu3fz6quvEhsby5QpUwDzyO655x7S0tIq5L6VJgd5EimqO7Wck3TKNb5bngM4F0uqWYyFJq8PAEzTMU6yloNrFqa/6p2/Ehjjs/4eTJwAfLhN5/lBf97T5SBdbJUZx+nA1rt3bw0KjdAbX56s174+W5ctX6EXXnhhbn9JVdX09HR98sknK8y+lRUH2TopRef9ts+vQQBwkFW5DvIGYLiqpgD1sRZTpWZidir7WwfTX03BwqgdfV6bB3QXkZoiEowlAuE8LileoGJwIOU952KrODhOB7Zjx46hqvy6LYv9GTkQ5CE8PDy3v+SxY8fYvn07CQkJpx3bqc4VhaNEJhWrm0el4iCl4ASXnVjvxGDsgLkTazL8LLAP00xtjnkK7L61AAAgAElEQVSR64AxwEXAA6q6REQWAZ+p6uMi8jhWuzgemATUxGocH1bVSSISh4VGf8bKOvpgEnCPYWUYa4HbVPVgIfinY6UgA4E2wDEs2aY6FlqdCjQFzgZ2YHWX+5zL5ziPz8E0WdsC4aqaXdSeuRyki60y4zhd2Gb8uhbNzOCMM5ry2Qfv8upLL9C3b18GDx5MdnY2P/74I/PmzSuWlwyUfSsrDrJN+w76/pcz/Fp7Zlx0uddBVjYPsiB91NHAdWrqNsHAnar6FvAFMERVb8DqCX9S1RRVfRH4CThXRKKxYv9uzv3PBWZiPOFVqtoR68/4vOT91GoB/EdV2wKHgIex4v2OWHPjvxfzGeaq6a9uxA7Di5y5RFX9G1Yv2REYhB2Qz6v1j3wGEyzv5nifozAv+QQTV4vVxVZFcZQVtuBqMYCy8fd03n53NB6Ph6CgoNx148ePB6BevXq0b9+ekJAQ5syZw969eyvEvpUeBymuFmt5DfLVIwLJwEyf1y8EPnUejyavHvF8LCHGu64bMAHzAIcBs4FIIN15PQR4DViCcYVHsMSaOGCdz30uwzzYxc5YgTU+Lgz/dJzaRedz1HEep2LdPJKxmsrFWJbrRvKiAHdjWa7e91oFDCtuz1wO0sVWmXGUB7Y2bdpoRJs+6omqr7WvfU3r3faBJiS20XHjxmnDhg111KhRqqr6+OOP6y233BKQ+1ZmHGS7Drpo/X6/Bi4HWbqm+fRRsaSWU7F5WFKM12NcBPwFK50A88zqAmeqeWvbyKtzPORzHwG+V/NMU1S1jareXtAbioWH2wEfiMgy7EA+VyzEWwv4FONJn8EaJP8IRAFLRKSV816+fWQOY+UgfplIxeFAynvOxVZxcJQHtt69e5NzZA+eGg0IrlEf8YRwYZ+rSE1NZceOHfTo0YNjx44xceJEevTocVqx+TtXFI6SmJzECASrbBxk/nrEuzEur4eqrhGrFVykqi/L8fWIZwIvqGp3n3tNx7jMdsDlGI/5nHPt/wEJqjpIRC7ADqZmzqVfqYV3EWtptcDn/asBjZ2DPD/2q4FXgcvVekWux0LCF2Aeq1fAMBmTtZuEcaffYD8KXgB+wcLJX4tILBCiqkU2TnY5SBdbZcZxOrENGjSIzp078+233zJ67Dh2ZEZy1nX381T/FCZ//jFTJn/D4sWL2bJlCxkZGbRr147BgwczduzYgNu3suIg27bvqOO/nunX2uSmUS4HWcrWDvhFRBZjGqUPY3JsH4vIUqzB8H8LuG4JkC0iv4rIvc7cT8B2VT3iPG7i/AUYB3Ry7nkzVn94gqnqDqwl1XgRWYIdbK0Kwb4U8xQHici52OG4XlVXq+py4CD2g2aLDz6wAzhOrSvIF8BHIrIDO1AL/J9ZXA7SxVZFcZQltn/84x907dqVr7/+mjmrtrN6bTpjH+xH07h47nriTVSVNWvWkJOTQ1ZWFitWrODzzz8nNjaWuXPnEh8fz8qVK9m+fXuu9xZoe1QaJuLfCAgr7xivO/IGdkDeCMzADvjC+NN08vjJTsB0n3UpwPdYF5FWxb2ny0G62CozjkDC1rjr5SpBHk0Z8ISec//bKp5gTU5O1p49e+qQIUN0+PDhOnDgQA0KCtKBAwcG3B5RChxkm3YddOmGA34NAoCDdJV0ytkkrzQlzpl6HPMyY4EmIvIrVu94E7BURH7EEnY+FZEbnWtaich/sfKP3zBPOhz4VkRuVlWvt1kUDqBicCDlPediqzg4Aglb9tFDeMIiqBGbSFBICEGeYKKiokhLSyMtLY1p06bRo0cPgoOD+f77708rtvxzRe1RiSyQCEY/rFJxkBXFxPo7ejnLaKAGllDzJ2cuEXgb+MT5Wwurj6yPHaaPYZxjDywZ6SushCQdy8iti2W0nq2qGUVhcTlIF1tlxhFI2GakbsITEsaRPdvIzjjCkV1bGHDrzVx11VX07duXuLg4OnfuzO7du5k6dSpxcXEBtUelwkEmd9SPvin29zoASU2qlzsH6XqQp2giMhgTzz18steq6lU+92kJfIcdlH9T015NxxKCfhORFlhoY4CI7ASuVNUxYlqrj6ol9EwGpqnqGOeewzCpuQIPR3G1WF1sVRRHeWPLPpZB9tHDZGcdIyg4hLCwsOPWBwUFUa9ePTweT26tZNOmTYmJiQmIPSqpCQHEL/pj5R3jragDHx7wJK7xFDLvyz0+6ty7mfNaCCbaC1ZTGVLA/GjgWp/7DcPaYxWLyeUgXWyVGUegYlu1apUGBwdrtXaXqadGA/VE1dOGN4/S4OiGWrt2ba1fv76qFlwrWZE5yLbtO+iKzQf9GgQAB1nZsljLxESkmoh8LZblukxEhmI84DQRmeaseUMsM3S5iDzmc226iDwjIguBawq4dztgqqq+j4VRH8Nk8q4TkbUY9/ibiPyMHYqLRKQ+Vou5Q0TGAr2Be0SkrYj8gtVsDhHzPv39jEDgcEeBgsPFVrFxBDI2VSXn8B5CohsQmdCVw2kziUj4A7t376Zv376F1kqeLmyF7VFJTSqQko7LQfphYjWKl6jqX5zn0cCvmOrNTmeulqruFhEP8ANwj5qWazomPfdsIffuhdU0rsbCrAex8oyJWAnJCuAOTKygKSafVwcrTVkCnOesmQR0x3pPzsP4yxzgbi0iScflIF1slRlHIGKbM2cOR44cIScnBxUPofVbUqfPA+yaMoJjO34jKOsIkZGRhdZKVmQOMim5o34yZZZfa1s1rFbuHKTrQfpnS4GLHU/wXFXdV8Caax0vcREmFN7G57UJhd1YVacAY4F/YiLnD2MKOSuA11X1LKzH5BQsGScIWKCqF2Li5F+o6k2qOhGrs3wQuAr4o5om7QmHo7h1kC62KoojELA1atSIuLg4+vXrR3iTdkhIKNs/+SfZB7YTWrc52dnZ1KxZk9jYWMLCwhg7dizz588nMjKSqKgovv76a9LT08tlj0rDxM8REFbeMd6KMiiYJ/TWIjYD1gA1neejgVudx7nrirj3TViz5J+xA1CxjiF9ndenYwo7YFmq053Hw3C4Rhx+E1PXuQdIA3oU97lcDtLFVplxVDRsdevWVRA9//FP9ZJXZ2lkncYa1aRFbm/JrKwsrV69uvbv37/CcpCrth7ya+BykBXDxCTsDqvxhCMwabcDmKcHVqZxCNgnxg/2Pol7V8PUfu7DyjsewQ7Iq4H7nWVxWMeQZcArPpc/CPQVq5V8SES+A35T1VcwwfKXTgIHEFj8TCDgcLFVbBwVDVt2djYSHExOdhZBwSGE126IJzQit7fk5MmTycnJ4ZJLLilTHEXtUUlMBIJE/BqBYC4H6YeJ8YQjME4vE+speQ5OBw1VvUBM27Ur1i/SG/ocLcZB5nKVBdz7aqxN1yVY/8oJmNe5WVXbOofzr9gBvAvjHw+oapKIKPC+qt4k9n/yNmA31o6rHvAPVR1f1GdzOUgXW2XGURGxLVmzgZBqNdDsLCTIQ3RsIv996A4GDx7Mli1bOP/887nllluK7S0ZiBxku5SO+vl3s/1am1A/stw5SLcO0g9T4wmn5Juej4mLe9fcWsi1cYXdV0RiMNWbi4EPgFRV3Sci+7CEG4CzgG9V9WbnmtsxjhMgG9N6RVVVRF7Guni8i3GhHxfyvm4dpIutSuKoCNg0J4vso4dRzXFUd0Jz1x4+fJjk5GQAYmNj+eabb+jUqRNr1qyhQYMGZb5HpWKB4Rz6Z+Ud463KAwudLqNofvMK4D2fa27HOo+AiQH43q8RJl5+J/CsPxhcDtLFVplxVAZsgwYN0vBajTT+j//Q6ISOGtmwubbo/4j27NlTn3/+ee3fv78mJibqwIEDA56DTEruqGu3H/Fr4HKQVctE5HURWewdmJfXEuu8kQysBe7FJOUecS77BejnXLMc4ypnOK9VE5ERYrWXU7GOI2dgnm36SWIDAoefCRQcLraKjaMyYNu2bRvhdRqzf+0i6nbsSd2UC8nYvYXU1FQmTpzILbfcwubNm+nWrVuZ4igtE/FvBIK5HGQ5mojEYRmq+4FqWNJPH4zLfBpYraodReTPwP9hGa71sC4du8Q4yD6q+q2Yvms1YAxWMpKj1sy5SHM5SBdbZcZRWbAt3CnsSZ3Dmf/6kN0rZnPg9/9n77zDrKquNv5bM5Shd0TpIDYEqVYsYNTYS+yKNSZqojExJkaNPRqjiRo1xh41UaOfGntXmg2lCSJGRVRUQFSK1EHW98e7z5x9z9yZuZTAAHfNc56599y991mn7bVXe9dkutssPv30U5YtW0bTpk25/PLLV5tf8n/lg+zdp78//mJhPsiubRqsdR9kUYNcS2TyPx6LzKS9gUeR//B24CRgBnBTaN4BBQgtA+oBCULOUgQcAMrVHI6Ch/4K9Daz1lUcu9o8yHnz5jFnzhwaNGjAgQcemPQp+omKvK3zfKyrvC2d/zV1GjVjwvWn8M3k11j8zZc0b96cBg0aYJaqW3PnzqVevXqUlZVRv3791c7H6qD4ONVttYLWto13Q92Q//F9YFL4/mfgp3na7QaMAhqG78OA3cLn76J2FwPTgRFAfSRQa8SKzfogW7Vq5W3btvUxY8Z4z549/Yorrlitvo311U9U5K128rG+8ta/f3+3klJv0mojb9i8lWPmbbtu4Y0aNfIFCxZ4eXm5d+/e3Y844oha5YPstU0//+TrxQVtFH2QGzT9EUHHbWZmV6P6j9ea/IlHmFl7M7sT+AEqfvwvM3sY1Xzc28z+AGBmx5rwV09FKSaDvYYSV/koWbHVqVOHxYsXs2DBAtydBx54YLX6Ngrdlzyga/KYRd7WPz7WV966d+9Olz47cMa9Izj43GtpuUkXWnfshpkxc+ZMli9fXmE2Xd18rCpZgVttoKIPci2Ryf/4JMJTHYQ0vusQ7mr3sP8epGluiTTJRWH/otD2EeBl4BDgfGBP4O/ufo+ZLQfaehX5lwllfZB9+/ZlyZIlTJo0ic8//5yLLrqIzTff/H+ac7W++ImKvNU+PtZn3mZ4E/Y96w988s6bvHDLFbTutClzPhhLWVkZy5cv5/vvv6dnz54sX7689vgg+/b3p19+raC2HVuWrXUfZDEPci2Tux9tZtcCE939TuA6U4WOh1DKxsPIJ/kboAXSFF8BXkPBOOchcHKABkC38PnTqo5pK5AHWVpamvN9bfti1jYfRd7WbT7WJ97mzZ7B30/ZC1++nI26b0WDJs3Y8/DDufnmmzEzttlmG0pKSvj222+pW7cu9erV4/PPP6dx48aUlpauFB+rh2qLflgArW0bb23egObA6SvYZzfgyQLadSH1P16LqnEkuKr3kmKvTgF+hYTkr1E069vhtzOAK6sYfxor4YPcaKONvGnTZv7CS8N9yy239F69evsd/7i31vpiarOfaEPmrbbwsSHxdsYZZ3jdZu28yyl3+qa/etxLGzbzRpvt7GVlZb5w4UJ3d99666191113XWs+yN59+vn0b5cUtFH0QdZ6ag6c/j8aO8ZyHYnQcczM2qASVqPDb28AZ6Hgm5FISCYVOl5COZJtUeeWZtZ5ZZgx06quvLycZs2b0bFTJ6ykhMOOOJJXR43MaZOv3+relzyga/KYRd7WPz42JN5mzpxJnaZtqdt8Y5Yt+BZ3p07jFpSXlzNlyhQWLlzIJ598wsCBA1eJj1WlEitsqw1U9EFWQ2b2AEKyeR9hsM509/3CbzeiFc4/zOyHyCe4EEWcdnP3/cysJXAnMnsuBH7i7u9E498H9AaeQbmPPVFR5HnAhe5+mwla7i+oXmR9YAvgSGAsSvGYgQJ3FgFfhf8NUU5kb19BH2SbNm2Y/N57NGrYkNmzZ9O4SRM22XgTOnRoX2t9MWt6X5G3dYePDY23F0e+SWmDJlBSh0ZdB7C8fDFHDNyY2267DTNj66235ve//32NMQX/Kx/kNn37+3PD3iio7cbN6611H2RRg6yezgU+ciXcn5OvgZmVAbcB+wP9gXbRz5cA41x5juehoJsKcvej3X1rdz8HeAH4BGmLPwAuNAGV/wH4D7At0Ad4HkiE3qbIzNoQmIqEcx+kZY6uSjhaDXmQzZo2o0nTpnTq3Jndd/8B7Tt0qNW+mLW9r7bwUZt5+1+MP2nSJEaMGEG9evX46KOPVjpnt7Zco9XBW/223fDly/HyxZTPnYEvW8obb7xBt27d6Nq1K+Xl5bz88suMHTuWBg0aVAjAQvlYLbQuhbGubRtvbd7I9RPuRuRbBG5EQOF9gBHR/gOSdghKrlv022dA0yqOdTFwKcpzHICE6UGoMsinqHzVeFR38uTA2wdR/3uAY8LnbqFtnZrOsYjFWuRtXeVj8ODBvv3223vPnj3d3QvO2a0t12hN8Lb33nt7Sb0GvsWZ93ivC57xuk3b+JAhQ3zIkCHu7r506VLv0KGDH3TQQWvMBzlj7tKCNoo+yHWKlpGrcZeF/8cjkHBM0aiXh89DEC7qIDN73czGAq2Q6RMzu9DM3jKzSWZ2axhrCyQc/wXsh8ytoNSO5UApsJ+734EiVlub2WgzGwe0B5aY2QnALUi7fKnQkzOrXf6Z2sJHkbfay8fWW2/N9OnTKS8vZ+nSpQXn7NaWa7QmeFu6dCnuTt3GLbGSOpQ2bEaDBg34+OOP+fjjj1mwYAGzZ8+md+/eBfOxKmRW+FYbqOiDrIbMrBUw1t07m1lHFByzORJO45AJdSryBfYC/gFshQJrRqO8xPbItzgQCb5b3P1SM2vp7t+E49yLhN9WyP94CSpZtT3yNS4GtgaOQzmTZwEXoUjXjibYuk9QQFFd4EpgjrtvWdM5FrFYi7ytq3y0a9eO/v37c+qpp9KhQwdOOumkgnJ2a8s1WlO8DZ/wEV6+BCsppaSsEcftuzP77LMPhx56KOXl5QwaNIgzzzwz57r9r3yQffr19xdGvFlQ27ZN6q51H2QxD7IacgGCv2pmk1AgzYOoPNXHSEACvI7MoM8g/+NHQDNgZ+AJJOxmIw1wBtIqAQab2W+Q/7Al8B7SFA9EwvEyd//CzBajQJ/XCVojqvSxM9DGVBUEJGBbB15GIXCBvGTFepBF3tYTPuLfVjZnt7Zco/8Vb3UaNad8zgyWL1tK/SadKS0tpW7dunTp0oXy8nKmT5/Op59+yty5cyvyJb/55psKDTIea3VQLVEOC6O1beNdHRuZuohr4fgvAWciH+KhKCBnGgrcuT9P+zJgJtAxfL8RCc9xSPscELWdRlobcgAwLHweA2yeZ+wTgBsL5b3ogyzytq7ycfrPz/COHTv5pj16+NwFS7xXr95+77015+zWlmu0tnjb4/ATvE37Tn71oyP9jtc+9KYtW/uBBx6Yky/ZtWtXLy0tXe0+yG369vPZ35UXtFH0Qa59MrOV1qKjvkl+YpKreCqpsNvJzDYN7RuZ2Wak/svZZtYYCdXJ7t4XpWo0Ce2rW2w9B5yRtDGzvit7HqE/UHv8M7WFjyJvtZePmTNm0qlzZ+rVq0e9evU47IgjGTmy5pzd2nKN1hZvs2d8TuuNO9C2Q2eWf7+cevXrU1pampMvOXPmTEpKSvKOtWpkBf/VBlonfJBmdg6wxN3/agqE2cbdh5gCYU5GZsnrUWDLIuBAd59pSrr/OwIFBzjL3V81s4sR3mk3FCF6LAIP3w3lGt7k7rdUwctuwGUIQHwLd9/MzHZHfsjm7r7AzKYCDdx948Dj3ahM1UwEE7crMse2QKbXuihPcgzwb1Su6jukVTYJxzsDmVjruXsnM2uA/JVzQpv5wL4omnVjYKi7P17TtS36IIu8rat8jHr1VebNnUt5eTltN9qIPff8IV9+Mb1W8Fabr9uED6bx5bSPaNG2Hb58Oe06dWP7nt0pLy+vyJfs0KEDW2yxRU6/1eGD7NtvgL88qjAfZMtGdda6D3Jd0SBHIp8byMzY2Mzqhn0jUGToG+6+Tfh+Smh7PXCtuw8EfoRqLSa0FfADdz8KCdm5od1A4BQz61oNP/2AX7j7ZgDu/pK713X3BeH3IcDX4beXAx83u3tvd/+Fu/dx93buXt/dG6DI10vdfTAStHVQIeStgO2AC8KYWwAfmtlB7r4otDvO3cuQ3/FyFCy0CzL35iWrIQ8ytFnvfDG1lbfVlc9Xm6/b/2L83r23YadBu9C1WzfKysqYP39ereGtNl+3skaN6brVNixbupTypUsorVPK4sWLc/Illy9fzpIlS1i6dCkNGzbkrbfeAi3oNyxa2zbeQjakYU0FmgIvIoGzQ/i8FbCEVBs+Arg9fJ5Fmj84HvgcaIxyDi+Kxv8/hFSTtPsYWFQFL7sBr9TAbxdC/mT4/mvg4mraX0yKw9oF+Dj67UDgnuj7ySifckA476cRJN6lwPmhTQmKYq3x2hZ9kGuft5XN56vN16228FHkrbB9jbsPzMmXrNOoubdr1847d+7sX331lV9xxRUOTF/VubxP3/4+Z+H3BW0UfZCFkbuXI6F1AqpiMRIYjHL93gPKPUgG4HvS6NwSYHuXxtbH3du7+3fht0TbAwVWnRG16xrGqYoWVPMbgJM/Z7JQqmn8hMrdfR93n4NMtUsA3H05KxihbFa7/DO1hY81wdvK5vOtCd5Wdl9t4aPIW2H7fPky3NN8yTpNWjFv3jyWLVtW8Uwid86qUTV5j9mtNtA64YMEMPkNTwrbRFTiaYy7H2xm37l749DuUJRMf4IJ63Scu18dfuvj7uPDWP2BF1x+zReBbVDC/84oz3AI+f2aByK/5vTAWj6/5mcILu5+pOluCbzs7vtXcW7/ALZ1963MrAvyRZ7twnndGAX79Ed+z+eAjYATEerObKRN/geY5e6HhDGXAue5+zV5jhenefTv169frfPP1BY+1gRvK5vPV5uvW23ho8hbYfueG/+pAM6jfMndenXm+eefp6SkhNatW/PFF1984u5dapysq6F+/Qf48FdH19wQaNqgtOiDXAEaiYJPXnf3mSh5fmT1XTgTGGBm75jZZBRdmtDHpH7N+giMfCwSaptTtV/zDGCqV+/XPBKZf49GwTcPA/2ter9mXnL3LxEm7CvABCQ8v87TdBIS8gnVQQE/+ca81d0HuPuApk2b0rFjR/773//y0UcfMXDgQBo3blzxPfGHxW3WxL7awsea4m3w4MH06NGDjz76iPPPP79W8bYu81HkrfB9DTfuwRY/v4vNT7+dVv33o2vXrkybNo0lS5Ywbtw4gLZmtgurSlbgVgtonQEKcPeXSKHX8BAgEz43jj7/H/Ip4gLrPiLPWBebgnzeN7OmyDT5EPAAihg9E6VpPBm6jAH2CJ97A19YmqDf1JSqAfC4K3gGdG2/Rn7T3uEYPZBgztI/EGzcJsA1SEjH/N4f7zOzYcl5m9m08PlUM3svjNEGLSQ+y3OsHKpbty6ffZY2W7ZsGcuWLav4Pn36dLp27crUqVPX6L7awsea4G2jjduzZJns8ovDsO3bt8+5L+vadastfBR5K2zfsXv0Z+rUqTx38e4AXHnlaB4av4jzhn9Nsh5v0Lx1g0VzZm+LFIaVJqst0q8Q+l85N9eFjaoT/I0IfCD89o/weTZQlmesiwmBNuH7w8BeBfIxCHg6+n47cEI17YcRwATIBRK4NJzPFcCZhRy7X79+3rVrV586daovWbLEe/Xq5R06dKj43rt3bx8/fnxOmzWxr7bwsSZ4GzN+kk/54GPfqmdPX1Tuvqjcvby8vFbwVluuUZG3Nc/bPpf9y4+6/TU/7l8T/Kg7Xvc69Rs48MNVmXP79uvv3y1ZXtDGag7SQVbBu1H1pWMK6rO2hdSa3pCZNYlW/RaZav+Agn2WAv8N7RYGYbgw7F8EvAnch3yTryGz5iIEEfcV0hIXIlPoH5FfsC5wNYqS/ToItDiy9kSgY9i/GTAZaZlJYeSxYdsx8PVAOF4iIL8LY3RBmu534Zz2r+Ya/AR4G3i7U6dO/tRTT3mPHj28W7dufvnll1f67u5rZV9t4eN/zdthRxzp7dq18zp16vgm7dv7zbfcXmt4Wx/4KPK2cvsOvvYpb9FpM2/RaTNv1r6bN2jRxn0V59++/fr7gqXLC9oKEZAIhnMWUdZA2P9DVMf3Q+DcsG9oMi8C/y6E33UmSGd1UzCxjkF+wy9RbuOrSHs8B/gn0MndvzGzw1Hy/p5IEH4Q+pQjjNSfIszV+e7+RzPrgUyizwPHoCoebyE/5svAdi7fYszPn4DDEJ7qCygf8h53X5yM5+4DzOxg4GYUPPQOEshdgG9QJOtbKAq2jhfg4B4wYIC//fbbK3bxirRaafGyyvvK1hnnR5HWVzr+vndyvj91wVHMnvruKtlH+/Uf4K++Udh807Ce1RikE3yi36G5cuuwrxQpJHugYMq3gKNQytwzrkDN+9z96Jp42JBfw+uBRxHw94/c/StgMzM7GdjF5d87NUR81iGttvEuMMHdd4oHCwLuRjObiFJENgsCrQEw0d3vDO2GIzCCHJQbd/+Nmf0N1ZI8xMyaAbeZWZ9kvND0GaTRTgT2Riuh6aH9jaFNWdS+WhozZsxsM/sECeZsgeWV3bc6x1rd+2oLH7WZt9rCR5G32sVbZ1aRxo0d81zDeta6wOZlZhZL01vd/da4gbuPCJH/MW0LfOjuUwHM7AEkHKcDHZDlrrAA1VVVmdfFDeVTPhUuUr5E/L8AXZF63iLs/0fo1wt4Nc+YF6MAmxIkUJeF/dcCJ0Xt7kVlqvLx1YW0QHPe8cJv96DCzPclY1XXvsBrUsmcsbL7VudYRd42XD6KvNVe3mrTRmVglkMJYDHh+1CkPDRClZJupkAf5LqU5rFayMz6I//esa6E+tHArmbWOqjmRwHDUfTpAmCumW2EtDWQXXtjMxsYxmtiAi1vBnwZxhyKNE6QH/EIMysN5oAjgCvMbHzYqgImbAbshfyPU4HS0H4vlL5xIkpTeTZqn+/4RSpSkYq0wZO7L3D3E939NJ3R054AACAASURBVHf/VyF9NkQT689R/cVXApLE26R5hgY85e6PAZjZOGAKSvx/FcDdl5rZEcANwXy6CIEC/A142MyOQ0IrQcN5FIEFTECR/EPdPW9+YqDEKfw3YPfA0/3Az9y9T+CrLtJEH3P3pVH7fMcvUpGKVKQNiT5HgY8JdQj7VpzWtnpc3HJMBf2B4Wvp2D9ZXftW51hF3jZcPoq81V7eatNGZRNrHWR164qqKE0Aeq7M2BtsFGttIzMbgHyK57r7I2ubnyIVqUhFqu1kZvejAhKtUTnBi9z9DjPbB7gOuZrudPc/rNT4RQG55snMeiETaUxL3H27tcFPkYpUpCIVqTIVBWSRilSkIhWpSHlog4tiLVKR1haZWQMz23xt81GkIhWpMCoKyCJVUEhFuSb6flX4f9hKjvXL6HuJme24Cnz9soB2JQF8vqaxCgrxrmaM/c2sxncnpBRV9EEJysPD9z5m9nhVfVeAl1UWuibqWHPLIq0smdlOhewrUu2ioom1SDlkZm+4+/bh80RUiWSMu/eL2pzs7ndk+t0LnOruC8zsWATdN8RDakpoM87d++Y55k7A+NB3KEI56uvun0RtRrv7tnn63ofwdb9HkFIdgavC9mzg/37gfHefb2YXoHqf+7j7aDNrWcMl+RbBBXZz90vNrBNwC0Iqehh4xd2fMbN+efr+CzjM3SeZ2RjgJuAGd28UeJ8IHIQqtS8xs90Cv/egpObO5KZidcrwcSiCOazn7l0D6tKlCB0qKeU2EqUi3Qxs5O5bm1lvVKptGYJTPMXM3kc1SJMKNphZ9yp42xpVppmL8oXLEB5mTGe6+27xjvCMXIUQqZIi4keg3ONFpPfrWeBnwHwE3N8f+IgUyWV4uLZXZ87pAIRlPAilS41CyFfnhL7xAm0qGXL33oHPfPcybjc2Oiej8vPRBRjl7svNbDNgC+DizDvUHfg/d+8bXds9UK3ZZ1z5zJjZfqjCUPIsWDi3Lhm2+qBSfHG7rh5VOqpmvLpAd3dfuVSI9ZiKArJIOWRmNwPtUfmvH6GXtozcvMqGwFJ3bxj63IQmuVboRf8Hmth+hwDb/x36n4XwY//s0YNnZu+gWpZJ31JgYSKoQ5tr0YucjJXQne7ex8yOQUJ5d2QZuQjh1f4KmOHuDcxsEHA5Eqbbocn6rGis5kggWvj8KZqslyNhv6WZtUAYu7uHc74UTbYtkJD4Phqvfrh2RwNPo1yspq4ao8l5L0cFr7uENo8Bh4RrMDkarxdCf4r5mI6KfA9LFh5m9gXC5U0ioQ8OPBwD3BK1m4tA+o8LAuafwEB3r9BGTSXdsrwdiMAyNke5wQ2AweH3mPZ399bRWKUIv/hdJCCfRuAbZe7ePGAMZ+/XXmgB0Dicf6JxDUX5zPtmzulrhK+clIY7IvD/+7C/Tdh/OIp4vBdVd3ge+CUqNmDhnBKKa862RkXRy0M7Q/f4NirflzbomXgbPRebBj4S+k3g4UfRtd0FoXdtj96/u9A9PwRBS05FwjEfHmp7tGCaiJ4pUFT8Pu4+N2lkZh8m4yXvoJldFK7JN+j9egilSNyAir3XQ8/jAnev1kKz3tHazmEpbrVrQy9lst0d/n+SadMAAaofFdpcD4wNv10InBw+z0cADMm2DL3gS4F54fd52b6o3tz3qBzZ42GbnRnrFQT8/i4SnA8Bu6IKKxOQgP5hGHdR+H8lElYXocn6omgbgyaT5Bz3Rppiwtu46LcJ4X8rJGCnIYzcD4AzMtcqqdDyFXA8WiD0QJPP36Pxz0n6omos9TPj5ONjQZ59i4BG0fdGaLGRbZezDwFiONLU3kET7aI8vC1EE3Q81jvR59+F+7os3NvkPn+NBEVJdP02IpSVq+J+XY8E/Pj4ePn4j66bRd9LkrEyfcfl2Te2gHfjQ2DLAu5LwtsZSHO/CAnV+HmbHp6D+Nom96IZsop8hrT0k4C6NfA2Ks++x9Ai7w5UbOGv4bglVYzRGy2apoT7timqi1uKkLuuXNvz05reNkQknSJVQ+5+YvLZzMa6e79gGiNjjvwx0g5fRZVMnjCz3wHHArsEH900dx9c0zHNbHjcF03SdyLtLKE/u/vwPH3PRAJqAhKsIxBm7nLgNDNrAyw2s1uQNnwVEiKHAFe7+8IwzqHuXqEFucymfwIWBu0nWW23ARqb2aNoArkH2NbdZ5lZQ2CyCeA+Ns20BJoAfwr/70Nay2XAcDM7CgnP/UP7ciT0l0RjlOfhY7GZHY1gCHugWqALydVivwe+Dya9pO+hwJKABJXweQoS2gdGff+Thzfc3c0sGasRUGJmf0H3DuBWJKh+FY2VmMmXm9my4CueFfiYEu5Jcr++M7PnkRbzO4R01SwaZ6fQL3tO85AZOjHNdwQ+NbPTEaJVcj3rmNlO7v5q6LtjOId+5Jpn30FaahfSggV7o9J41d0XzGwHpLWfjATOr9z9kugc9kHabXxt65pZK/QeDEXC6XUkUC8ws6QYAeHadXX3y4JZ9z4zux0tKpPz/ITUkpDQN8DTpqIJFc+Xu/8ljDkDLWY6ufuHZlbq7t8Dd5mQxX7HBkRFE2uRcsjMytBL3RNpiBOQ6fIEtBpOTDyOJrWEStCK/y13Hxle2r2RKS6ZOIejVexGpD4oUGmaozN9d3P3e8zsWHf/p5nlTLYJhRc7ew5tgG/c/fsgtDYC+iKz0gfBD3MTev47mdk2wBNoYv9nGOaYwPc9yFTXD2nLhyLt80J3r1RZ3cx2R5pGlp5H5dLw4Fs14fkuQNrC68hH1gT53BqQO9ltHj7HfFyKzJV7hjbPIY3jWCQQQD7OJxDc4Y7IhPwx0l5PCf2fR+bLE5C2m9ybHkhgvu7u95tZVwT6PB0tNq5E2k0DpK3cHfoNRWbXfT3XJ90aCfEjgbNRmaLx4fPc6H41AzYGprr7HDPbFd2b+ujZ+wY4H5lj43Oqg7Sg0ej53BY9l4nJMfGT1kVCoFkY79tw/QeRCpSDwnV4l9RsuTcy9/4hui87I7NrfF/+hYTfq+5+lZl1C583Dn0ws60I9z26tk8Efu9FBdq/DAuF75DZ9ZbQfb9wjrFZ9wO0UHw3Ol9HvtxO7v5+OG4yXmyKHRjOqw2yxDyIno8fIM1+BnrmT/DgHthQqCggi5RDZvYQWvEejXw5x6OJ7f64GVIkTqphrIeRyTOZOK9FqBfL0cS4PZqYLNY0zWx7Uv9HWTheOZqQE+qNVvjzM4ctQZN9okUNRy/7VqSBKz8F9gEe99R/9R4SFIkwHwFc4qoHugUpLu5L7v6embVDE7AjwT6jimuwTTjuX5DAeCHs3wW4yd17BU2uU7ge1dGbMR/IfHo+qYZD4OdEUn/dSHcfF47ZCJnX5ofvrcIxDQmLi5FPcxYK4njP3XvmOac9SIXy80gT75Npswj5qmOf9FBPA8C6oACf94DTyF1E/R1N2D3IXUiNB3D3edFxKs4pCNIqKWuBMJWIw93nhiClbdx9cfitAfCtu5dF7e/KNyyyDOQ8H9lGGd/+IrRQuM4DalawuJzn7pdn+k3yUOcw2pdYdsZFz+/imNewb39U4ScO4hrhGT+imV2JyuaNj/Z1Rj7Sesg/2wz4m7vnW/ytt1QUkEXKoeSlM7N33L23CRj9v0hzSMxPr4b/OyKBdzESXvHDZEBDdy+Nxk5Wre4KrNkCuAK9fId4CCYw1YA7Ek0mA4DjUH3N30Vj/dTdbwkBBjEdgExhicY5FEURNiHVDn4N/BaVIUsmmAnuvo2ZNQn8fRdMZ++6+xaZa3QyMnu9HM5zV+BSDzU/o3a/QFraI8DpSGBcjib6K5Em0IfKk9il7n5AZqx8EaVnhPOcRKoNgDS8jVAB7keROXwOlelFUuH6F+QD/iXSRm5GUbrTYzbQPd6TaHEQjnGOu48KvO4EPOeqqXoh8LkL/muRu8cBMASzYF1ytc9NUZBUdxTY1AVpRzdHXfcBDnX3OWGcFigK94LM+HXJFcDDkCa2J7KSJELlBKBfNF5zZD05xd2fz3PtqOb52Aw9Y11IFy5bIA0/5v8Zdz8h6lcpyjuY+V+MeTBVANoRLcz6BYvJFGBnd58ctRsDDCE3iOsrVOrpeTNr6u7zLH8k935IaC7J89sGQ0UBWaQcCn6ibc1sBJrUZ6DJ6QFkOgJNVGXIvAaa1L539x9nxnqd3InzPTRRNwC2C5P9u8gk2RcF/ixAJs1/I+E7LAx3ZOAhh9z9zMwxJ2TNQEGbae3uC8L3R1FU5DwUzfoLpAG0Q/5CUFDQ8cicdoa7fxqN9z6wo7t/Hb63Al7zKAI07H8H2CGYGR9BWvHTyLy1r7t/VcUkthgtSmLqhrSwf6L78DgKhsqZ3MzsDCS8Z6KFx7fIdHYzuXQgEqqJSW7/MGZ/YG93/8LMJpOWeUvoCBRBGi8O7kI+3cRk+Q3S4B9G2uwuSCv9GtjT3d+K+K3qfrUgNQUPRM9CYmLcH93PlqH9KHcfZGbfkxvhnESZ3k+uAN4eCfbB4ZoeirTZOugZdGRC/gwt0Az5dsvCb7eQ0l7AnzxKezKzCYHvMUT+YHcfE7W5B1lIHo943h8tRB/xMDGb2XxkKVhCGj1bihY3sVm3EdA2XLMkGrebuzfKaJrfh9+WhvNdFPpOIzc6tjW6jyPQu/isuy9jA6OigCxSDpnZj9HE1htNfI2R8Iv9JxNQVN1W8b6ggbUlXZVvhYJikomzGVqZ/gAJhW9JUzdiOi/02w1Fis5FE9F7UdvDkL/sLFKfaRnSIF9x98MCX92QEGgRmc/ah33JRPI8ijb9rbu/EtrshrTbZUh4jyadyAYB7TyUGjOzekiQ/5FcreEcVM8zmSRL0MQ7Opw7QFt33z4zic1CkYf3hTZHIi3wL0ho3+zuN5jZB8gkGfsqbwK2ToR3GK+Nu38VX2Azm5y5fy8iv9uVaHKchQTTIeRqi8OpYnFgAaQhaCXtqOxXfgNN4p+Ea2lIW9za3T+K7tfEMLGPRwuYF4A27r5laNME+cRaJRpOMIm+nTUJVyWAXWkkiZWkMRJmV5BL1yDA6+nufreZ3Yo0wYfC7z8K31sDXxBMwMAu7t48c8wOaIGUmL5no8jneVGz89D78D0SXIkro1JqRdbsjwR4lq5CAvjcwOuZ6L09NU/bShS0773RomgQ8EJ2Eby+U1FAFqlGMuXI3ejub4TvU4DJ7n5I+N4NTWLLyePDiifOaMxdkcB8xt3Lg5DZLPy8CJnWYv/HYJSntyz0r4ty1D4j9ZleisyOmyHBZIGPp9HElASuHIP8KddG/Hzg7j0y5z0BTSoJJYhCPdBE/xgSHAeG76PJDX7YAQmAJJduEMqr/L9ozOPQBBdPYke6e5uoDWa2AAnJu1CqwcdmNgdp33Fgxg+BjvFq38z+izSEfyPt5FszuwNFBk8ObRoR0iTC9UkWNWeTqy0uA7bKLA5GIrN77Ee81KP8u9C2M5VpRzSRTyW9X58hbeostJDqi4JcfhjGqY9Mv++H6wHSVB8H/ozMy8ki5SngoCoE8BtoAfANKpe0aYbfEShYLEnafwPYyRXVialQ+gSkUd+FFi8gc21iek4WLg+jgK+kSMGxyNS5R55rkkPBfBz7Y09GloTXEqtIaDcI6OHudwWza+twnD3DtX0ORU7vie5VW7QwiHM9K8jdx4b37IcES4BHua0bAhUFZJFyKPhejkOa0HFhdwlp4ryjSex7NCkmk9oytNp/EU0CmyJzXawdHg3c51HkqSmF5A5kKpoWxusJHOxRlKiZzUSC4ZvwvQXSSBbm8ZmOQr4/gPeDKTcJ4Qc4LdFGovHnoMk1nsD6u/vBUZusvzOm09y9XXZn5rjvkiahj0YmsyUo0CaexPYHfuzuo8MYAwNfzwMHuHsXU9Tj6DyC9A4U8foUuWkio5AmehDSvMejxcQMUk3aPaDJhLHymZKnhi1eHLREwuotpOk/CVxAbpRzYr7LBv3UQwIxMU+/jxhJNMNd0cJgG3IjTB9Ei6AfhH0voIVRYl5OFgyNSOsDJs/qy8h3uzvSuB0JsHbkovy0QakWz4RrdC4KrLki8NYM+Wv7Ak9GFoCPqUybuHv9ioshAfYuAhKIg2uuI9df2g65ADqQBrZNQ8FtO6AgtZEo6rcJsLm7b2ZmmwAPuXsOnJ2Z/RFZBv6FLBwbhTGyyEItwvF2C3w8CDy/oZlZiwKySDlkZq8hwTMRvSQJPZxpWo90sn8frfAHBK3rUTTpnEau7+unCPnkknCs0nCcRcDRnoaivwssdvcYyzTJbXsFTXS7oOCgnwWf6QSk+fRHGsnp0XHPdPfdTLlpOyLNZCapCbMp8uO8TCrMRobxB4fx2objVpi9gmbsrgjKq1AEY1UBHYcjaLRhYYydQ99uedoORHmgjUPbeUhIvIt8lw+GdnehCNI4MCOvEI+ueWtkqh2KhFui8b6L7lk8IdRHwuFX7j41aIsfogVNTKcin1t8nPFeObJ1ImmaUBnQFSj3ELgTBY1MQNc9pt5ISIIiMceZ2Uak5t/RwGvIt/113DFonLEAPgD51Oab2e+RL68LCqy5Fi1QTkTC4ZVoqL5Ii/4P6TM4EWl3TZGJ/SF3n5nhHTN7CWmZSTT4eORLtHD9jkfCeQGpr/+ocM6dgDc8Cmxz90OCGfvwwHd7tBAYGwnqKciH2IVUo94WoTklWnEpAiioWBiF/fejxe0zSHN8IXtOGwIVBWSRcshCCHkNbfJFBh6MJtwcH5a772gCATgPCdTEV5IECtyKJv3epqT0o5GQKiEAe6OV8XIkAJOamW+6+4zIZ3pt4GdjtOp/P2J5f3dvHbSR3dCEVIq0B9AK+glXjmRFFGs41w9D/4rQfVNx67sCXyAf6T3IxFtCGkxR4T8Kk/4e7j4rfG+DNPIXyRVKyXhvo8l0iSsNYScksDuTYmi2D+3jwAzPM9k1RffnSBQZ+igKxIm1xcuQ2fK+MM6RyNzbKHy/C93fd8JWD5ndv7MoGMsUEbktMuXtlTkvIgtAO6TB3YjMqIa0pwuRpjydVOtMEvF3jobaF2l0w0gXHAuAPu6+zMyGuPvLZnZIlgfgL0ELHxT4vAZ4MJhdJ7p7r8DjmHiRFvZtHM4P4EAPqU4mLNgjkJn8c1LoOAKPz6BndIewry4S+k96igG7EGicEV7z3b1h4o8N1pBvkT9+JlrIjQr3YltLU0AaIT/nr8gNFnogjJPch5YoQCznmcmcc41zwvpKRSSdImXpXjM7BZnJYqSNb6I2N6MX/G/h+1BkJluIfIaJD+s2U8ToTqR+xR+4e5w6gJltZAr3fwH547ZFL/WfQ5P5aFI+kggU2sy2dffbQ5sTsicSC2Yzm0c64ZYAf/VcZJOzTQntSWRkEsU60yvntd0JnO7uI0PbQUj7HECEcZmhkkQ4Bkq0nKnIlBfjh26OzIUnAo+YGcgcehy5k10HKqdv3GGVq4QMQebPiz3kAppZExPQ+xPoPg9FmmKSV3qrmV2CNMPTwr7HkCZ9DlrsnBau0x+Am4LJsT1azMwOvMbkKBoXJDxPQJpXYnL/Hj0/R3rIDwy8Jprnk+G4XUmB1meZQCQ+QJaBYWb2FLCbKWXmSKQdx5QsbPYFbnP3p8zMTbmIH5jZz9Gz2jgcfwt3n2IpiPln4f8OZtbPBV4eo9D0RQI+fj/6eJS6YyoK8KmZfWlm+6Ign1LkykjetWYIqac50lpfCMJxSWg7J7SdDTxoQotqHt7fk4Cv3D0netmUdjPOzGJLzLmZ64Pl5iI3tBAh7BsYFmtRgyxSDpnZz9CEN4dUs8kxBVr+yMBZ7t42s+9jpFXFfr3jUQ5i7Hd5EyF+xObNmzwEgoSxbiY/aPgepD7TOiivEDShJLQ7quYxPoxVaUVsZt8hTTEbxfoW8gP9h3TBcE3WNGoKx2+WrP6zZGZXk1YWAU3cfYDP3H1gpu2nCJXoHDRJ7YeqLeRc3yqOcz2VBe48dC9bory+76xy0vu+KJ8y8ekdigTm9qZ0lZ2CSfI1dC1zrlOwFFQKxsrwFqMhlSDT5lZ5TLH5tJW5CBN4Wfj9RU/TPC5Ck/1ppEKpguKFUGj/JBKAewQeFiFTaT8koC5Dgvtqd3/DzG51958EoVIxLPLlfYEiamMUmvvzvB/vkQKRO1oY/RQtCm4Ix3sZmUxj4fU7d38gjJEEtj3r7kvNbEu00PglKV7qnujdegU9b7PIDRbaBAnx5Jkb7XlALiw3F9kDjzm5yBsCFQVkkXLIzKYibNHZ1bQZi8o4xZGBkzxU94ja5SSGB3PoDShaMkbSeczdr4/aHYJyzeqR+v0aunup5aZDTEBmtcRnuhyZT7uS+nH2IwUI/wyZdQejyeiL0KYpcHge/iegQJCEkpzDTZFZ6340eRyBfFNfkwZ0ALlQeOG8KhBuUEBJGbCXhzxLUzrEc2ERkIA21Eca0r9RoEo8fswfZvZWHoE7CZl9eyIt5yvgeHefFLXphoTyDuGc3kC+xstRqs0SJKQauPtmmfFnouCQOGn/NRQB+2xkMoz9o8tQsMkoZFbdxN33NkGwPYFwVN9B974XEt7NUDDU86aE97HkLgTecfffhmOVIA0wX1rCIqSJJmb1jYFeXoX/OIx3A7pnzwY/6e/D+T2CrBExCk2+92MiejYTfoei8nDbRYdJTLg5wssqR6cegCwMuyCB/kbg7WlkfbkV3a98i7X27l6vqvOM+HjbFVPwDrDMM8g9GwoVBWSRcsiE1XiQBxDvKtrsjnxSU9GKtAtaCX8QNWuCJv9zSCeFaWgV3cJzkXS6eG6tvA9ROa04Ty8fesjzSLuN+45AVTkSH2JjFLXpyDf6X2TCnIIEAsiEexLSZKuLYo01iCx1Jk1GT6gF8sPuFPxLiX8yIUdm6XooXcCQcD8d+a1OcffrgsD5PPAXk7v7kHhH0FSyAncK0hD/4mn9wStRqkCMJoNH8IFhcvxZxpT8FPLZxdfp11457+8DFDgTl27q7e4PZdqNC7yc78qjrYMWGju6+7uhzVUo1eAJpCWNQIuVW8i1OhxGbm3Qpmgx0Cj0hXTB1AUF1PyJAigIvTouX/kg9Ny2R/mse2aa74wWgklkaBeUS9w9Gu9D5C9PBO8olIKze+a4U5GJOI5OHYPMySPd/YvQ7sdooRGn5Fzh7rdmxrsXCfS3qIbCe5RgsW6P3CobHBZr0QdZpCwtAMYHYRBrKnE+4KtoctodTb6PokngN1Gb+UhI3oCCE0DP23Go8kd95KvZA1iW8Zu1QUEqMf01HGcjM/sDMgFeAHSwXJ9pOwTp9l3oV45C2ZuiSMC+ZlbH03zKe919qAkbtAtpKsEI4CQTbFi22PABnsHMzEcW6ima2abIJ/UY0NPd94natEJadFKXMkmXSJBUTgnX4zfuHldzqIrOBkaZ2UekAncGElZ3A7j7sGCea4dMdJeitJhOFrA/w3m2ToRj6DfKzKYFfh4JPI4EPjez+p6btL/U3Y8x+SWPQsFILcKC5Z/uniwUNnP3B03+YoIJtV4iHAMtDLyWo0XGk0g4vZnxVV4UtLtjkCZ/LhJg7aIF00VIyO+CBE1BAjJQ4vvdF2mnl6H34G1S/zbo+euBnqeDkFY+08zORUEyjrS8zkiIHoaesTZhMZSM1RQtQLsTLBkuhKOv0AJqYDA3j0YL0b6em5LzeThOTIcBR5miwhOwhmboXYrpUWS2/Tky4XZEAUgbFBU1yCLlkJkdn2+/u98dtXkQ+bUSM+bRQHMP6DVRu7uBs1yJ6WORSTNJbh+CTK1N0YQRBwr8FAnT2HcC0gSTFfbLLtDwrM+0OZqgrkAv/37hGIOAa8Ok/TGpf7UD8iG1Qyt6i34j8HAOuYV5J6OE/zgx/ibkA4s1soHu3sTMzkFpKzfkM1MFjS5GIGpFKqiXoWjFVuGcYlPkDh5BnEXj1UcILyCBez+aYCtpfZbmj45AGu+y6Dxnowjh2JS8mFDxJDHvmtlvUWpENmn/jnCsn6H72RKh6CTn1xQl6ndFKC39TMEhj4fjJtCCRyDtfyjStHZHk3a2yO/LyK97H4rqHG6C7WuSCORwbSa4+xY1mQzN7AVkKp0Tnt+vUCpGOanvcnQef2NyTeMo2YfRc0a4lqXIzFsW/pcjS8KnpM9g4tfuYrnRqUlazzDSCN6lyEy81BQh3Bn5IXcKbTZBi8bbSd+hhC4hBf0vC7+PdfdDq7o2GwoVNcgiVZAprHxPdz+mhqZbx+ZP4JUgNLLU290TSDVz9/3CpHRs0FArAg4yfJyABGNsunJkok1C/hPf5tnAprHP1JSGkfj6TkU17F4DXjKzv6JoWcL+09EEXkqu1ppMUl+7+2izWEFgEzShHB6+D0Va4HlIICd5bVtY5XqKdTPnehFKPdkK+ZD2RsVvr7UUtm8TpP3djEAFQKbifxNyEq3qtIbuSBgNJM1lHYlqbgLMMbOtkbbfCPkZE6pPmnyf0DZhrHJgSNB4hiBfZRLgc1m4BicgoXwa0kAuQRGrFyPtbT4Sqo8D3c3sVaSd7osWH4lW3Tlcl7pIc5sMXO9CaUrSK4ajezaNUBvUhNwzA3jTzB4LY+2PaifGgqYqap34VcPYByNf6F4oGrc1cItVDipKfHxxlOyXqFpI4r+8NIx1gbs/BhURptdFbfoB06xydGodpC3GEby/jM6zN7LM1CWNEB6ABOZCD+XWIjoh+RAWv/2Az4KJPSZHC5LrEp7XdypqkEXKITMbhSJFl1bTJgs9tx3yVR2XaTcBQXUlGmRjlNfXK9MuDimvh4RVTkh5mDwOQ5O8IdPVQ2gizfGZmkpM7UJqAsyJkkwooxV/5RlUmrD/GWRmeiis4A8F7vTKJYMWuvLV3vE0r20impTimn+Hu/tVUb+JSOiMCz64jRCaTiNyYfuWuPBD4yClimR8M7vE3S+y/CWZWqKAjS6ki+LmaCLthcpRtUeT9o+i8zzZ3XPAyqvQgPPtRw9nywAAIABJREFUG+xppOvr7r5DbNrOtK2D/MKGkI8S6MHN0T38p8tnfQyavM8FxgQtrR16Lo5EmmI2BzSJbE4WTK+6e9Z8n5dMQPIHu9IxTkD38tFwfarzR/dCz2kcJTsn3L9Eq3wYLe5GIyCAD4DB7t4xo3nehhZYMdLSfcmzatVH8G7p7keGduPQ+3IaqcujgjwEk5mChGYj18mQbDu0KPiXZyqYrK9U1CCLlKWpwKsmn2AFzqO7/8XSfLS6wGumdIQEem5KnrH+DLxuqjG5MRKQv8/T7kZyy1udA5ya8YedgTBGE8DxP6JI2Mnk+ky3AbYOYxoyB97q7jckB0tW/OF/SThmEtGapZ8hP84WZvY5Ssr/wMwGeW55p8Q/Fee1bRz7bl34qYsz4y9y9+VmtsyUJjELmUfbo1SGvmY2GGk9rUh9k9sTaXvuflH4f2L2BEyQcb8mtyxWPaSVdUHaaQuknWwcnecx4Vxis/FGZtbJ0yCgzuSapBN+Xgma6VZA//CsYIL0Ay0AFiAza5z+sVkwH/8cmYcN2Cpcg4PQwqzczFqa2TDS9IpTgI/M7OhwTkkFjAYIwagCSs3MWnpuXm+OOTV8bxHOa5SZJdrpJcBPwvllkX7isRqioKJrkLY9CfkDIdUq7wsa4V3IRHos8pXHbZ4ys+YuFJsXovHdzJ5Dpu9PkAb9D8+ks4S2yf0zZKloTpoHmtDRJjM/6H3YCgEnZDVNgE/M7Nhk0VPVNVhfqCggi5Slj8JWQuUXab8VGcjd7zHlUw1BvrT3gctMhVwXZNp+aGalrqrye5OW98Hd3zEFfpQhHxjI/Pc5yk+Mcx4PBs5199ugIgLyWzP7ZdQmqUzyRuDrY1JzaZYOQqbPV9A1SSb1O0z+LFAgyO9MASlnk+a1za08HCeQRs8CvG1KBL8NmR2/A75z96/NrMTMSoKwmU9lU2QlH5GpBuVdyHx5G9Jgyt39iUy7ZwN/Y9C1XhSux2Skcc5DoOrvkVsW6h1yhcbOBKGRGT82HS9A9+5tUtCBXyJN5mpS83NC+yMz6XJ338OE8vMcCvRJTKeO/NtxekV8Tj8K5/8LJEAqmpELWJBQbE4lWD1Kw/VLClmf5dWkP0V9FxJ8yGb276BxTgmm0j2Aq4J2uhl6Xkcii8dNUZuvTOk59TOmzibIFxpH8N5KnmLbZvZ35I8fjKwypQjgPJsXOiz6ugdKP8kB88ic39tmVlbV7+sTFU2sRVotZCEatKp9JkirXnk7y5yUhJTPQNrkt2iCTMyJc5HwiOv1jSYU9E00taDlDow0zUeRqS6BKStDJrmW7n5hFedyfGJ+NaHNDEDCKQn6mYTMdkkljbliwS8NfWLIvCQKdFMCiLZnQvmj43YhRZbJV3pqFzKmyDxjJGXH9kJ+wAvQAmI4uWWxrnP3TlG/Z1GgU1ye62x338Ryy0I9gyb1ZEJ+I5/QiE3HYby9kam0kOoVyfHucPeT431Rm0rm2sTiED6vEDxaxpy6BVowPItQoXLIM7mnNYyb5LImWuVEV+5lT5STGBdCrmiDAmWSKNsToiHnI8tCFugi5/rE+4KAXYaen2fcPYbsy/KbD0Sjkra4otd3XaWiBlkkAMzsOnc/y8yeIL/J7IA83WLK1uErRcDhSf9vzGwfD4ncUburUJBLCWlIeQI5dk1ocygyJf056jqsCj7uQsEKSWmrvYDzPBfA+rowIeYVkEjrSPyTHVCl+ThN4FMkbHZBuZhZeg1FLLaOeL4NaZc5gQ9mtguV6U9ogo5h+z5HOJ1JbmALMzvK3bN+pySaaB/gHnd/N5hm+yDTeGJiXWhmvdx9YnKeHspJRbwdHrVNEFi6IrPwkyZovvPM7Po85rgK0zGaZ2ahoJ4EZade4GcBWhjFEcETTNCDMyzN2VxkuUg8kAagJPRadE4NLT8iD5BXyJ1PqhlvH/hLKrzkdCW/b67KQ4XjVWiV4fu7ZCDw8mmeRKDtZnYaCirrlkerfDXPsZNqKguRybmc1HpSFVmefRuEtpiPigKySAndE/5fsyKdrGq80wSIPKY9EMxcTHu7+2+DCXVjd7/ElMyc9fsd4O7TwjFbIH9kNsou8ZUOIzU/fYLMcslkmfgcq3v240miLbmpJuVAI3c/LGgH2QmUICw+IQWmxswWVKF5nBN9LiPg0EaaRaLJjvcoqT2YAE+hcmDGGBPYQ1dk9m2CUIgG5Jygoo7HmFJelgCbmNl/PRcl58lg/r0aaZaOrt9CUyDUr1AU7T/NbNcgEDdDPtSxken4d6H/PYmP1MwMgZ9fReWI4O5I8zwWBW7NQbmhWZN/lgYBJ4Rz2hTlshL6xlRJyLn7s+EZ2R5FB+fVjNcg5RNU9yEN/kpy06LmZ32qgeL79wCK8L09T7uY8pkU8+3Lx996R0UTa5EAMLOX3H13M7sqq+UV2P9KrwKnMV75kqYXgCa819DLew1Qz927mkCmL0UJ5iUuDNBhCGKrDjI9zUIRib8Kx2hJfnoF+dMSc1wCcXaNh/JaefitMB+Zwu0PRkn+kPrLTgd+4XlSYsxslLsPCn7D5AVrhFby7lUAPof2IEGZraXYECG5JEE6pQhaLau5lyBtcaorf68VEmLnuftkM+sZtMrOmcO/iIKt8lYGMflby4BXgk/tQuBzd7/DhBLUGgX6vIoE0tLk2iSm43wLGgvRv5l9lUplFUKZc3oKBbskC5aq+mSByCG31NbH2T4egRMUwNMjHgqLrwitrAkzmMFJLB7R/h+jSOx8fvG4XRwlvREy7V+HkI1mRe229giqcH2looAsElChUfwYTaZHk1khFuJ3MbMDiEr8uPuTYX8zNHnmXfkGc+eQ0KdvWPVORoIz0fSOdPe24UXv6EppiFMqkuT/bKJ/ezSRV6q7WM155KQtWJpX+VtkEixHSClTqabMVHVjVtPOgHc9N88UE9h5ZxScAQJT+Mzdzy5gzPeQVvYxSin4MMtvRrjEptavMsNdhiKDT0T3ehZKyWlgZmeE49yDIiyPyvQdTCpwEk3+Z8iKMCr4LRsgM2C8kKqHUl6WoHv7OvBLd88W+U1SfHZG92o/lD7xK1T54ydm1gPBtiXP5m3uforlpm0kKQyOgoNics+F42uITOedwjg5468sFfq8RO17oeveEj2PX6FnZO8V4c3MbnT3n1tu/dL9kZ/9HHf/v5U9p3WRiibWIiV0IUrB6EBl306NfhczuxKZBxN0nV+Y2Y7ufl5Ytc5FEFelKJy9DtA4rHjLXTUPk+GeRv6pBIAc4GBTjtbhpMnyKYPuXQMfJchv1xUJMEeQXFn/VZL0/LinYAYJ5fhzXHlzb5vZf6ia6lfzGygVoRKZQLATgZ5of/kWI79FE14SBfoCNZvLEooF3lPkiUaOtSxTCklV9A4SVCe7gLQ7IRi1HdB1NxSc055cc72hAKNE4CSa/P6kpbJK0XNyCLlpQ4+i5zOB2jsSuN/MdiR9lkBC+zDkx7sfCfJytNDaMbT5HN2LJ8N5nxL+V5m2UQPdhSwaiTk9Z/x8FJ7RQz0Uvq6C8vkUq6NbUPWVuMrKs8ifHfM2wlQZJ0t3I0D0nwd3x+9RsNssU7rOTGRl2KAEZFGDLFIOBZPijSgEvYw0yGBEDf3eQTXvqqxUbqqzdzEhmjPsdmSSewlplz9Ck8Mj7n5q1Pcw9NK+6u6nmSokXB36XInSCcqQqc9RHuJ1qPzQJCr7QxOszM7IT4aHKNSVIRP4dFLAdiRCw6nWnBX6xdB+y4Bp7r6ik2PBtLKmuyrGOt7d7zaVYTob3beDkKA8Gy1QBpECNtzsIbo49K8QFJYplRWZ9wCuymNK/hwtSuJnaVOUrrEgtGmEaiI2zJgOK8q1Wf6CyjEtoTKge8VzYmnVi7zjV3Pt3s76hVeF8h3TUvCKmLcvSAOPytBi6Xt0LVu6e/egaY5z98bRWCUIoq+qSPT1kooaZJGyNAMFN3QgLUn1GpXxG/NRtthrls5CJp44ojQxU52PJqP7wnEnBI0xCZB5yaNKEMG89iMT8s9FKKduMMrluw3VdrwljD/bc4sj/x1Nqp2RKe90pFmtNLn7pkGb2hn5vm4yszk1+dI8QvPJR2b2oLsfbilIQ7Z/lWbdNUC/AO52FWEeDmBmR7r7VFO1lfoIZB5ktp9iZn08zTdshoKMHvSohmTGvGdAexN60wWkmLANkAD91kOASrhGSYoK4bMHjSjx3XYnN+gqm4MZ0w4olSjOAx2dabO0hvGrohfN7NcoICgG5MgXbFMITQ2L2xhvd1Ee3qZ7FFhmZteg6OSWhGoxrjSURZaCEYCu+dMryds6S0UNskg5FCaZgSiKr6IkVU2BBqbcvz+SW+z1XHf/d9TmFWAPj/LXgqZ5lbv/OtqXr2hzHYQ/mlNVA+Wu9Tezie7ey1QWqw56tivKYmV8ihX5YeH/eOQPrTI/rCYysw5IOO6KTIzfIC3yyhr65RN8c1FS/eUocOlLqxxUA1QfgFLF8d5w9+pMqCsy1qzgF45Tg3ZBC6zB7t4k0z6nPmjYNxMJw1hQjEBwh7NCm08QmHyMdtQOLeY88S8HM/rxyCQL0mYnIRzYrVBKzk6obNOwAs4v+5w0JpNHaGZ7IMG9QuMHn3mWKs5lRckU2X0JueW/hqFFaZW8hX5funuZpTmbdZCZ/xKi+qXunlzXDYaKArJIOWSh4G4QGtu5+xIzezdr4qqib1Ls1VHdxhmZ3+9AfqinyF1lHx5P2panaLMpPy1bVWMSilAdhHwjL6MgkVOQcL2bUBYr1j7N7E13387M3kD+rqdRjuGmhV2lvOe+HJmKr/AVAHI2sz8hTee+sOtIFLE6Axjk7tVpOPnGy5dXWaOJfGXIzN5zFXbeNdp9G7r+5wMXei5e7zNA98Tna4o8nkEKw5ZQTlHf2LwX+ZJ7kv9ZGkaukLgHIfpsjxZuVaZvWGVYvZ+4e4foOfkGFQbfNLQ3ZGlZWMj4a5qC1v0OioieijTEV0gXM6UIkWlC2H8cgnQ8HZjs7pV8/RsaFU2sRcrSdFMU6X+AF8zsW5TTVwjtQOpzSspVxfRp2OqRVjwAGGfCfn0IaRHzELLOA1Gbhl65qsYyZOZrCJyJIiybAScjX6QhIPP3Mnxk8/taIe13VagvOvejTXX/PgCGe55yVBn6QcYnONHS0kbHWm6qSExJ5Gw2ZSRvXiUrltxe+WBRvceIFiMmhkftFrjKTP2dFK8XFD07A2HVzkU+4cNQsM+98aBmdnXGvHcUMMvMHkERpp8gDTt5lpqQ4q5OC1tCk4Bu7l6tCd1yYdkSc+pMq5wHelvSx93dzJ4OfrkVNtFbilUb+zfvqbpH3jHyAnsEaoWsLnuQ5pY+TFqWLCmlthy9MxMRWMeT6Dl8i2oKCGwIVNQgi1QlBc0gb0mqPG3/hoIkYp/FR+7+szxtc3K1rHIFiiHIn/kAqXZwAPKDxlU1KlWbKPC84uK+9ZFfads8AmBFx22MhGQCPo275zWNRn0mAKe4++jwfSBwuwsuboVC/asYvyOClVulYreR0I7hA5OUgB6kgVJNUORoHVJ4v5g2RRGl3yKNfyrSWAaFPi8Df0d+3MS8tx2Kav1n+F5RfzT4rx90lVJLUn0q2EbRtCVIqCYFgj3ru63CnPoWei7mW1p+6jKPUp5MNU9vdPcsGEG1ZFWXOVuhGoyR9n4IMjsn1+goJPx+jaw6gxH04CKPKnEEF8c9nuasJvf5bXILCByHilvnzXVeX6koIIu0WsjMpqDyOklAQAnK59syarM1CiJIkvpnoxfvNyjpPqmkcBp6wWPtqw1KV0gm149RtOSNVO3Du8WjqMmIj5xITjM7D0VTrnR0Z5hQ6qOAppHIZ1Oj5h0E4p2o0okh7fnHCIZsX68+FaAQvvLmVRbYtz6KKu6CNPURaJHy40zTPyDN41oU9HIiAnioCsovPsaDCEnnnyhVYTiZ4ttmNjnLfxCGi5BpvJMp//Gn7n56pl1BvlszG+3u22bMqXPdvb7llp+60N23i/pNQUK/WgGc57zzlTkrCKu2ivEqRcWakK0mo7zRJLJ6Vp6+FSXuIj9kEp0b5xqv8oJtXaOiibVIq4s+RGa0ZOLpGPbFdCuVc7VuAxp4biWFm83sJ64UgsTntJzKVTV+hDSQNuRqrvNRmsptwNAQQHMD0mhKgDomMO8kCf41JKhXhfZ292xSfY0UNI9epjxAPDc1ZIWFoxWeV1kIPUZaHeMhpEHVp3LkZxt3f8nMLAiei616rNuYKopvB/PsKWb2mZl9gGD+DMEYLnD3RqHddmhBsQsCkcfdJ5jZcUgbjelOrwIcPkNP5DGnJhGlcfmpyzP99ipg7HyUr8xZx5UcC6CRmXXzAJ5gqj1ajiAft0b3cY4JeHxRpm9c4q5NeOdamGpyjg9+8i/R87RBUVFAFmmVKPKBNAHeM7PR4ft2VA6Jb5QIRwB3H2bKVUui6W5xpTRMBrqacivbhub1kcb5GJo0h4bxN3f3gdExnrA00CgBg74LBcE8jqIct0MCdUL4fT7Ck10VWmpmfyEXdPtSrxnaK9bS6iQ+Vl/5nMy4GPAy4H5f+bzKfADmJ2f9qmb2WrAYfGDKdf0cCbBCaKyZbR+CeTwIv2bAnonv2IQEtLmZTQt9OiHT+9PApqbSSw2B0vAcJY7qpsjEWghNAb5394dN9Sj7Aa0st0RVfTJCItFEzawtKwbqna/M2esr0D9LvwSGmQLcDKUwHe3uz5nweE9A70E7KoNaxCXukjJ3j5FbQKAjek43KCqaWIu0SmS5EYyVKBPA8Shance5Wv1RMM95KMrxO2RevZHcIJ8HUUDL/DBWExQY0QbYy9MCvp2A50J0ZWIuysH2NLMfAb/3lcD7rIrM7GEUEJLkNQ4FtvGa02PiGoYVOXyeBwR9BXhpgODF8mLNrsA4twI3eFrxg6BVnEruQuAtdO7NSQOl/hSEXk3HeA9Fo36KIObqouCfjwimyirMpDcj4XIZMsVfEI7/CamAnIc0vxvz9M/ykfgeY3Pqxci3mpSo2hjo5bklqg5AifebIC2wM/CeFxD1HY3RhSqwaleEggBP/ItTUDTxzugdm0Zq+n+5iv4NERBH37BwXeS5wB/1XRVHNhxy9+JW3FZ5Q7mM1e5DeKx/RUJyDEK6aRF+S6rInwG8mWes99ELmnyvH/btgybXV1CI/yfIJNYIFbgFofQcS1o09l7SJPTbAz97ruL5jy9kX542k1bzfdg/XJePw/c+CE5vRcaYiNIDJiMz3fvh+0RkdrwbBVINQVrJ7avAb+domxT+3wk8EZ6FocgneAiyJnQK2zYI1nAmEkz/BH67CnyMC/+vRJpXxb4a+k1A0aJJ/8HAHQX0e6mQfSvA/3F5tvuRtaROaFMKTMnTd4dwrz9F8QHboIjjxlGbxqjY8mp7VteFrWhiLdLqorylrOJ9rvy3M/N1dvfJ6CXFzA4ys2aea568BxhtaZ3Hg4B/uPvTIYoyWTm/72lgznXh/0nIB3ktMv/WR8Wb90ST21AkNPPVdiyUFpnZIHcfFc5hJ3IrclRFcQ3D1UEXo9SOYQDuPj74o1aEKmG1RvSMu8fweC+b2X/DfelM5LbxKgJVMuZI91T738UFXm/InN4PRS+PQ5GYSV3JKrU0S1MnDiXFWy0kdeLzmsypVVC5u39tZiVmVuLur5jZdVU1jszBrVfBHJyPYjdDGQqmGuvubyY73f17M3vfzDol1zzQdciX+rgLyeeb8P59F/X9LmiYGxQVBWSRVoms+iKur2XavgAc5mm0agvgAXfPBjp8h/KwXiCC4ULRkUnqwN/c/VarjKXZ3czwqCSRy09UUfA5mNM+NbOzSYsKr2p9u9OAu5NgGxRpe3w17ROKaxgWVBmkBsoCv0PVeXJ5yXOBywchEOu7TKhEy82su7t/FH7vhjS6X5MLLl+JqjJHEopth8kZDzUjQ59xyIzeDXjRZf4bTEijyYwfp07UQWbgUaS1Tqujw1GU9DWuMmEbk5tTWhXNMaWEjAD+ZWazyH1ms/RThG6zCbKiJNVn5qNF3EqRu58Rfw/+zQfyNG0BvBtiBRI+N3P3zzLPzHIz6+chpcXM+lPYgm+9oqKALNKq0ooUcW3tudGq3wZtIkuPEFVfj9qPJURkmtnFYff+VM59c+ARy43ojKmOCb5sKWlR4Son9gLpPVQ5oTvyhc1FWm5NfqUVzuOsgd41s6NRwEoPpLG/VkOfvBQEzgDkI7wL+QcBXskEg3zg7o8XMORlCHGmJkF3N2naj6N7czVQk5Z2KCF1IvT5IWleYLXk8q3Fi6ovUeRmTXQgEhy/RGlHzVAt06qOcz1wvame5nXuPs/SHMtVCdLJ0gK0qMjS7/Psa2qqjOJmVhel9LwKPGQCNzcU3HPEauRvnaCigCzSKpFHpawgx3zW2MwaZ0w5y2PzTgi+yAfAfXcIBtki/P6+VwYqmG8KR59EWgeSzHhJROdOSKtIcGH7IHPuj919oamo8ImsGj2G4O3GUhk6rUrylY+CrIrOIBf4/TkkmFaGDkYIQWMB3P0LE6ReDyQ0Qf7JQWZ2O/L1bk5YFHjlwsKFmiN7xwspBKbdn5q1tIrUCeRvW9XUiULoSGCEu39AGqBVCB3q7pcGDX0ICgq6GfkMV5hMKRoJlaDnvVKakEdBc1HfQ4HrkYl3IXpPjkJBThX32d3LV4a3dZmKArJIq4XMbH9URzKv+SzQ+cAoE66qIXPpT/KMtQ9KGv8otOtqZj9192eiZkkawebI/5Kkf+xPSC/xUCkjmIEHeQBJN8GKTQIuCGal4e7+xCpegkopEYVQTWbHlaCtSE2MdZCGcwCwMibbpe7uZpaAPzQCcCEOVWjGZnYiWszURf7LxwlafGa8Qs2RJWbWIvisDaUoPEPNWlqcOnEdEuyrUyvLR52AW4Kf9210biPdfXwN/ZKI5epyLFeE2pGahJehgJufF9LRhR2bIOmMc/djw+cYCq9fcF2sEBTeOk9rO0qouK0fGwVG8yGM1P3C1jrzW8/wfwqwabS/O3mi78JvI4Am0fcmaEUft3kf1bpLvl+LJuaTwvYCAhlflfO/FaUA/E+u2wqM9z5aJHQlihBdybF+jRYqU1HKwP+3d+ZhlpTl2f/dbLKNjCSICyowLIGg4MBEkC+KCIRETICICooiiI4ooASifhgVox+7EiAuICJE4iUkIAgSEBUYyIADwwwMmwRMxIQE4kIIO3h/fzxv9amuru4+3X36dE/387uuvrpPnaq33rN0PfU+y/0sBg5vO2ft72EzP4nM4lUIw/1ewv27PSU7mYgfHkGUkdxDrHxPLn8fOMa5n0SsRPv1/V+rzP3nRD3laPtfXntv5xKJY8sncP6lLdtu7/LYDYgyq7OI5KhvEDcXPyayhM8lslr/oV/v53T5yTrIpCeoI021HHitw9U1auPYxhiVDuQS14r/SwLNTzxYEKB67l7iQljXVr3d9pa1fd5HZHf+mFiR7E90avhmeX5Ic+cxzLlqV7Ua4Xp8gDEk2/TifWuMd4Pt/zP6nl2PtzuR7SuivvQHLfucC5xs+y6N0JC5uMS/Y/vfa9uWEXHOjYnC/0uJ1fPRdBRtnmdwjLhyp59djvud2nMfa57XNe3UXiPpU4QLf10i9nkDsYIcMX5ZMkL3ZIQayy7PP5AkR3hcKuYQNY1DYrwtY1TyiLcSov3HEN6gTeiRFN7KSrpYk14x1my+NqoL3y2Svk/EUEx0fVhSZax6cGyrtfyjPqgjA/NKOvGdBRSJskJbc+duGakkoht68b7V+UwtHjggvu6h8cCuKAZxiFEEUMiW7UQk3iwrmbivrG4aWm4O5gBXS/oVEee6CPit7eck7UOIEpxRMlfXBz7kTvbsurZ/Vjv3cqL4/XtE/KzqMdoUWDAT7GQyCvuWc19BiCYsdhei9x5/UlCTsSTJDcfatj8OIOmTDjWhj7u3UngrJbmCTHpCiU89RRi5Kk50ge1fjmGMagXZ7O5Rx7YPbhw3n075x/W2b2sZ++V06vTeTKTb/1OZ75Dmzv2iF+9bY7xvEfHAO+msuoa8Z+NFgzuhVEpFmxMZwRCdOBaWk1YJSFU8sRrjNURG5J8T6jkLifj0W23/TNFEuZIR3ELSy4guLjvXxrjZRTR8pFVrPygGZGeiZGc/4OFeruJ7TUmO29z2NQrVpeOAax01xdvYXqHozvN/iSSkvyBKr5a5VoIzG0gDmUwbJutCJ+lE4oJcNxprAH/LMM2dV1Yk3Vt3L0/C+AOfUe2GZsTPrfm8pJcQhuSdREz6KmLl9e2S7HIjkVG51J3m2ANdJcrjAwiX9tVEjOwAIplkV0Jm7wPFcG9p+/JevgeN17YNcXP2RsJV/CDhYu1GqL3vSDqUSIxbn44Y/JpEbPhpOitxu/R+VI+k8FZG0sWa9ITi/jyRzj/dcA19R+KZMtYmRLnCxgxWZvnT9sNGZW/iQjng+irz3YXhmztPKhp7I+Ru+WdJWzuUiXpGMWovJzprvBYGumzsQijDjHh4GeMwoiB/A8K9emhznmUF+Qt7aPZsg1cTCki7lnmdSmRcnkq0RIMot7mIoqgzSZxAuMdPJ260pnspxIcJpaWbbc8DUDRF/zhh2Aeai5cM67rmbhrIJBknJxEusrtH2qn5T+daeYXtHcuf3yUEqL/HxAv4IRJnVqfE5DS0ufMHJe3mlubOk4XtOZM0dD0e2Atlnoo/IsotNiISOCh/H8XonVCqG4FXEPq4yyRdWJJ67mDojcLvKmTf5pYVz8GEZm6d/YBNHT0Ml9p+U0l4OknS/hBxvpLgNZlcY3tQLaekIx2CANORp8t7BoCk1Qh93ZcAp0uaRyRHzSW+OxeU446QtJPtiXa9WalIA5n0iv/qwjgeT9y9jvZP95Tt03s4tycIo1ElrryTyMg7t8zrPML9OhMYcy3mSNTiVecpmhvva/upAxibAAAUHklEQVSC8tw2tld0O5btT0raVtESa5miyfFwSU5b0ClU/3RL9mzVPeRhOsldz5SYWrXynEctUWmSeA8dzd+Kg4jEoenIdYoG4WuVDOXDCEm6LxDJa28iYsIbEsk7VTeP84gs3VllIDMGmUwIdbRQ30jchX6XYbInFVqt23lwC50h5RWN+FJ9rHGl60tqaqJ+jNBg/WJ5/lXAmbabjYBnNfV4le15JaZ3FY0sYRi5f2UtmeeIMl71ndgHOMv2GY39T6yyKmvbTiIMUXVT/x3CeC4hvAzPEivaxwhX69VE4sxBtq8dw8vuirJKPYCIP15fe2oOkZnbTZPmvqPo23kItdId4oZxHaLOdRFRqnINsEuVCStpfSKRZ6KeiJWKNJDJhKhlnNbl3ioGZU8WAznqP11ZaR5I1HXVMzF7kq6vUPJZQKeh8wJCBeXRcqLxxjpnFIoaxSpeVSXLPETIokEkd+xFdNY4WA25PHckBdd3dOm4HdjJ9uNl+zqEUMKvga2IxKlVy7Gr1uZxOCHucC+d78PahOu1zrnEZ7kj8V28yaES03MkbUlk4A4pryDqcJ9rPXAKKTek59t+V2P7lwjBhqeJBKnribju5+nUDk9ZpvdUki7WZEJUad8aLDBddepo1qQdD9wmadA/XcuwA/GlXsyxrHyOpyObtSahm/qZXow/g2mLVz3iWjNnSacQdav3MUqXDuIzf74zPM+XY/YkkmnOJlx4L9bgzjBbARfbfnt9co1yhbWJFdCmtq/oxYsfhW+XDN773aJvOh1xtLt6laQ16v9btj8GoBDtP4i40XgFoXZUtZz7BFEOkgYyScbBIIFpR6eO19Z3KGn819LpXffxYcor6vGlXnAuYQy/RMRYPkDIgV0naQuibvDKlSADsd+0xauamrVrE/HClzFKlw7ic7hZg0Ud/tv2v5TVzQWEms5SQi6v4luEOxMYuPl6G51yhXnEimdv4B2KTi2P07sEpTbWKKGAnTS05dq4hRn6wAPAjQpx80qQYmeivGN7QnzhG0Sd8PuBH9k+DkDSDn2f7RSTBjLpFXWB6cp92vb92qD8Xg14vRq9GwtzgXskLWFwDHK8rs+1bP9Qkmz/m6Q9Y4r6GhGrWkLUSb5rxFFmH58g4lV3EBfM7xOJVZXxWoUo6/kvd9Glw/YXyw1SVUT/PqL10xrAMkIs4CHgwfI5HVX2+ylwraQriO/DUYTx2wS4uYx9n6QHiWzbfrCQ+L7MZbAxh3ah9unC/eVnFSJeCiGT92ngVncE/f+MENQ4XdL3aL/hmfGkgUx6xanAYkkXlcf7EZlxA0j6BtFVYpDKC0MvJr12fT5dkhPuKxmUc4k75n2JxssnKaTLkholmers8gOApKuJprt/SLyP3wdOVMjlLWJ0ubyfEe/9aoSRO4GIO36ESJ56BaGwA50L+M/Lzxp04pTPt7h/n3Gt2fNkYvsGojPNLbbP6cc5e0FtNbi2Q+5uOFSM5WGSDiISd17UhylOKzJJJ+kZkramo3v5o2YRuKS7bG89BfNaQMTE5hIdIvYhxLDfDxxi+05Jd9h+db/nNp1R1FI2LxAvJFZ5FxMGbm+iD+JpjCKXJ+mviRjX/bVxu06+qi7qkpYSWZa/IcosDifcv3fZPnaML3NClNXvQgYX1H91urrrJe1E1Biva/uVpdTmg7YPa+z3Qdtfqz3eHviweyRZuLKQBjLpG5LOAU5tGs6W/eoqM2sQRf6PexzqMiW2daLto2vb3kjoS95o+0RJmxIF7EeMdfyZjKKRdMWahFfgk0QiTD0TdTGwO53Y8k9sD4kfKzqvvLqeIDKMEcb2prV9mhf1u4nsytsYXK7wdff5gqYQhl+dTrPkA4nV7fv7OY9ukXQzEb+9rJaZvML2NlM7s+lJGsikbxTDdBnRW64rlReFD+3PgB1tt2W8dnPem9xR6alvH83NlDSQ9CTwIttPlcdrEjFCA9fCQCPsY2z/Q+PYfyQ6dDxc29ZmhNd3Tcu0eVFXdPtYfTpc1NXSmqxt23RBReRdpTa1bJu2851qMgaZ9JNziDvsO+hSQq6sCL4r6TO0l4R0w20la+8iIja2BaFJaaI9U6ubabaj6JJSsQohxv0rhmairkr0sny4HLcB4QIdZCDplPmsYPjkq9Mk3SHpC7afVmi9bkjUF1a8GXhI0gPNOddXnn3ieUnzbN8PULwRz49yzFTyoKTXA5a0OnAkEX5IWkgDmfSTR2xfNtpOjbT56sL81ATOuybwSzrx0b2AHxGGEtvLJb1hmGNnM/U61ueIEoBdCdWVeibq+Q2X6i+Jz63JeYSgff0GafOaIa4+680Iw7MZ0eX+aSIZqLqoH0z0Xzy0HDew8hzXq5wYRwM/rhnrjYn3ZLqykJDBezkh5n41cbOYtJAGMuknt0n6e6KWbqRmvvW0+erCPK4SjxKDvN32l2rbbrb9juKqq5jOd/1Tgu03jfD0gOyfpH+SdBUd8fd3EAatyRNuaOxK+iydGsfqs37AtSbKZdyfEnJy1UX90EYS0GmSbiXKFfrJ7wDbEIZxb2AnOsX1046iLJTlTF2SBjLpJ2sRhnGP2ra2Mo9VaFflGXMGXVEP2Z8QCahIN1MX1OoQW3HRsrV9TFn1V6vKs2y3tQ9bpJARvIzODdJtwC/oyBQa+CNJfwfsBrze9n9L+o96zFHSfIWSDsT3YgVTcz37K9sXKZomv4mQ4fsK8LopmMuoFPf3oQxtJTerslO7JQ1k0jfcfTfyUVV5xsiNks4kZLIeB74MfIp0M43GDkRmauUWfyuhX3tffaeSyXqp7YsVGqVbSlq9pdSh+gzrCVNbEfHFSwkj+Vai7+DmwA2O/pCbAH/XGOtUOtmvOxAdKfYb16ucGJXn4S3A2bavkPT5KZhHt1xK1KteQ3pNRiWzWJO+IWkjwmW2c9m0iFgp/qKx33JC1LyuynPdeOsUFdqvdTYi1Fp6In4+U5F0PfAW24+Vx3OAK2y/obHfrUTm6ouIgvJbiKL9UV153Z6j5bi/oCOQfxQN3d9qdTvZSLqcuMnaHZgPPEmUuUzLrFBJy2xvN9XzWFnIFWTST84F/p7Onf67y7bdG/uNqsozFpqxNElL0jh2xYZAXTD+mbKtiUoB/yHAV4oy0bIhO0nrESpJ9aL6l1bnUPSbPBB4nQaLlQPQKAfanljdXkpcxxbSsrrtA28nxNZPsf0bSS8FjunzHMbC5ZL+xHZbjDhpkCvIpG+03b0Od0c7mirPGM/bTNzYkyhNOJqaJJrH2W9ypiLpWMIA1Es6vmP7+MZ+txFKNl9iBGWiUge5gsFF9W8nknMuITRBdyOK/r/cnE9dRq6+8iznfwNdrDxnKw3xjXWJGHDVksvjEeGYDeQKMuknv5T0bjrZjvsTJQFDKAZx3EaxQV0XdE2iDdMTQL3Jr+kY5ASw/QVJVxLuU4D32R7I/FVHnP6jhMLOJcU4bkoo3TSZZ/vPa4+PK5mqB5dz/Lp5jhGor24vYvjVbQLYngMg6VtEv8dFtjMxbRRyBZn0jZJ1eAaRCm/gn4HDbT/Y53m8ALjK9i7DPP9e2+e1PZd0kLTU9vzGtlUISbj/adl/MaGwc0N5vDPhmtypsd+OxPek3kT5OaL2sbqp35wwiiOubpPBKFqR/WH5mUeU6yyy/TdTOrFpShrIpG8omip/tJF8c0q/U8xL2cgS25sN8/yQC38ylEqurNS2LiSyIpcQguZ/Y/vkxv7bAucTYuYQK8b32r69sd8twDuJleEORAeQewglnroC02N0VrfXd7nynPWU2uAFRFnKQuBJ2783tbOanqSLNeknr6mMI0Sn+QmWb3SFpDvoxF9WJXpSfm74IwZq8pKRqd7TrW3/j6R3AVcSkoC3AgMGsqwst7S9bakZpG2VOTBwaaJc6lgfAjaw3ea2zbjxGJD0Q0IJaTGRRb7ALcLySZAGMukn3TZV7jV71f5+jmjw+9xwO9PSXSIZkdWL4MLewJm2n5U06D20/VtJfwlcOJJhLDyh0kRZ0kmEis7uiq4edQWmNI5j53YiA3gbQvHnN5IW235yaqc1PUkDmfSTnpZvdIujO/18QunFRK3eSO64XEEWilu0cmMusl1vLF29T18jJOKWA9eXWHObEbxG0tF0BBuA8CQ09juQwU2UtyQ+txMY3Gg7k6rGiO2PwUC96UFEmdVLgBdM4bSmLRmDTPpKL8s3xnDOTxPGuJK02xu4yHar4omkM21/ZLLnNd2RdCQhS1a9b/sQMnJnlOfXbzFu1bGrNVfp6qL34zBj/Qvhxn1mpP2S0ZH0EeKGZ3vipmYRcePzo6mc13QlDWQy41E06t3WnR6GGxLaq1UHhuuAz9metiLTU0Ep1t/JjebIjYL96v38f8DLbP9xuQnayfY5jf3WIuolq5X8IuCrTfdeiyHdkOgEs3EvX99spKzgFwG3jhJmSEgDmcwCitTcPjXx88uAVxN9BSFcetva3neYIWYlJblpgQc3R17SIgBwJeGqO7Yk4awG3Nay34WE6/WCsukAYD3bb2/s12yifB2hm3s9w/eRTJKekzHIZDbwKHCnpB8QK5M9gcuJAndsH9EmjZZwLkObI5/Tst/v2r5Q0icBHK2q2oSwt7G9de3xjyUNcbF7cBsrioTdWcQqNUn6RhrIZDZwCZ2CcogYzF0U3c5SsJ5ZfA1sf1HStdSaIw9Ta/h4WfUZBgr929zVSyXtaPumst/rCGHzQajTQBmi9dlWRI3l6ravkbQ2kcSTJJNKuliTWUe3BetJdxSDdgYh4XcnUWf6thYBgLuJjNSfl02vBO4lSm9cxTaLS7y6MD1HuFnnAuvYnidpcyJ2+WaSZBLJFWQy45G0F/DXwKuI7/zqRHf6jWDkgvWkK+4iVuhPEOo23yVqF5vs2eV4l9NpZQXRzuo04EhJR5WV7YsnNuUkGZ00kMls4DRgX+AO25Z0i+0dpnpSM4jzieSbKkZ4ANHgeFAD43o3jlGot7ISsD6wLeFWnVOSgNL1lUw6aSCT2cCDwAp34gndFqwn3dFV8s0Y2AiY704T5VWIBKFHCYH7i4DvTWD8JOmKjEEmMx5JCwgX63VEmcCx5am6a9WjFawn7ZQWSmc2km8+bPs94xzvXkK39+nyeE06Re0i+kV+3XnxSiaZNJDJjEfS1cD/ApVo+UbAg7aPm9KJzRC6Tb4Zw3hdNWpOkskmDWQy45G0wvY2tce32Z70LiKzhaK9OixjiD3Wx5xPRwP2L6kJBNTGzRV/MqmkgUxmPKUjxDW2ry6PTyHa/VycbrrpT4uyzn7A+rY/PUVTSmYJaSCTGY+kx4C1iQ70zwJzylPPAk8RcS3bfuHUzDAZK5Jutb39VM8jmdlkFmsyG1gPeBewie3PFZfg+4DflsevBF46pTNMhqVFWWcH8tqV9IFcQSYzHklfIfoI7mp7K0nnEE2Uf1Uevwi42vaCKZ1o0kqLss6/AqfYbhMjSJKekXdhyWzgdbbnS6p0RF8L/Cel+a7tX5cO9sn0pKmsY2AvKR7a/uIUzSuZ4aSBTGYDz0palc4qxIRxrMS1N6DTqT6ZfjSVdd4K/IQiNp8kk0UayGQ2cDpRU/diSV8g4o3/Dry0PH4b8KkpnF8yMk1lnc8CV9h+95TOKpnxpIFMZjy2L5B0K9EgWeW3a4/3tn33FE4xGZkNiQzkimfKtiSZVNJAJrMC2/cA9zQ2Nx8n05PzgZ80Gjd/c+qmk8wWMos1SZJpT0NZ5/phGjcnSU9JA5kkSZIkLawy1RNIkiRJkulIGsgkSZIkaSENZJJMMZKel7RM0gpJF0laewJj7SLp8vL3n0r6xAj7zpV02DjO8dnScLqr7Y19vinpbWM418aSVox1jknSC9JAJsnU86Tt7UpLrmeAhfUnFYz5f9X2ZbZPGGGXucCYDWSSzBbSQCbJ9GIRsFlZOd0r6XxgBfAKSXtIWixpaVlprgsgaU9J90haCuxbDSTpIElnlr83lHSJpOXl5/XACcC8sno9uex3jKQlkm6XdFxtrGMl/VTSDURz5BGRdGgZZ7mkf2ysineTdEsZb6+y/6qSTq6d+4MTfSOTZKKkgUySaYKk1YA/Bu4omzYHvmz794HHCbWf3WzPB24BjpK0JnA2Ib+2PfCSYYY/HbjO9rbAfOBO4BPA/WX1eoykPco5/wDYDthe0hskbQ+8s2z7E0L2bTQutr2gnO9u4JDacxuXc7wF+Gp5DYcAjxbB+AXAoZI26eI8STJppFBAkkw9a0laVv5eBJwDvAz4N9s3le07AlsDNxaR7jWIps+/B/zM9n0Akr4FfKDlHLsC7wGw/TzwaOliUmeP8lPVGK5LGMw5wCW2nyjnuKyL17SNpM8Tbtx1gatqz11o+7fAfZIeKK9hD+A1tfjkeuXc2bEjmTLSQCbJ1POk7e3qG4oRfLy+CfiB7f0b+w06boIION721xrn+Og4xvomIeG3XNJBwC6155rF11WnjsNt1w0pkjYex7mTpCekizVJVg5uAnaWtBmApHUkbUHI5W0saV7Zb/9hjv8h8KFy7KqS1gMeI1aHFVcBB9dimy+X9GLgemBvSWtJmkO4c0djDvCQpNWJZtV19pO0SpnzpsC95dwfKvsjaQtJ63RxniSZNHIFmSQrAbYfKSuxb0t6Qdn8Kds/lfQB4ApJTxAu2jktQxwJnCXpEOB54EO2F0u6sZRRXFnikFsBi8sK9n+Bd9teKuk7wHLgYWBJF1P+K+Bm4JHyuz6nnxPtql4ILLT9lKSvE7HJpYqTP0JoribJlJFSc0mSJEnSQrpYkyRJkqSFNJBJkiRJ0kIayCRJkiRpIQ1kkiRJkrSQBjJJkiRJWkgDmSRJkiQtpIFMkiRJkhbSQCZJkiRJC/8fjaUEnzyRkJgAAAAASUVORK5CYII=\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEWCAYAAABIVsEJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAIABJREFUeJzt3XmcHFW5//HPFwgkQJAEAoYESNgEwhIgbKJcNhERCQIqCF7CBSPK5npFvAoXWX9eBUGUC4ogsqNoVFzYvShbAiEkLBJCIAl7IBDAAEme3x/ndGiG7pnqmumZzsz3/Xr1a6qr6px6uqqmnq46VacVEZiZmTVqmZ4OwMzMlk5OIGZmVooTiJmZleIEYmZmpTiBmJlZKU4gZmZWihOIdStJl0g6NQ9/WNKjJeu5QNJ3uja65pP0RUnPSXpN0mr573p15h0n6Y7ujrGz2vtMJeo6UdLP8vAISSFpuS6qe50c67JdUV9f5ATSAyTNlLRHwXlvk3RkFy47JG3QzvRxkhblf6xXJU2WtE9XLb9aRPxfRHygo/lqHUgj4qiI+F4z4pK0kaRrJb0o6RVJUyR9tbMHGkn9gB8Ce0bEyhExN/+d0TWRN5ekXSQtzvvGa5JmS7pG0rbV8xX5TLmu2R0tMyJOj4gu2f/b/t9FxFM51kVdUX9f5ARitdwZESsDqwI/B66RNKjtTF31TbCVSFofuBuYBWweEe8DPgWMAQZ2svo1gf7AtE7W05OezvvGQGAH4BHg/yTt3tUL6o37V68TEX518wuYCeyRh8cBdwD/A7wMPAF8LE87DVgELABeA36cx28M3Ai8BDwKfLqq7kuA84E/AvNJB8P187S/AQG8nuv7TI3YxgF3VL1fKZcZA+wCzAa+CTwLXJbn2QeYDMwD/gFsUVV+K+C+HMvVwFXAqXnaLsDsqnnXBn4DvADMBX4MbJI//6Ic87yqz3lqVdnPA9PzOpkArFU1LYCjgMdyjOcDqrNtfgX8sYPtty8pCcwDbgM2abNtvw5MAV7Jn7k/sFFe75E/xy1VsW2Qh1fLsb8K3AN8r822KLXd8/RRVWWfA07M45cBTgAez+v8GmBwnc/9ru1VNf7HwMQ267vymfYGHsoxzcnrZiXgX8DivC5eA9YCTgauy9vgVeDIPO5Xua4Rue7xwNPAM8DX26yDU2vFC1yWl/evvLz/rKpvuTzPWnn9v0Talz5fVdfJed38Mn+WacCYnj6W9PSrxwPoiy/em0DeJh0AlwW+mP85lKffBhxZVXYl0rfjw4HlSAfoF4FN8/RL8oFguzz9cuCqqvJL/rnrxDaOfNDK5Y/P/zDvy/+QC4GzgBWAAXn5zwPb5/gPy59vBWB54EngK0A/4MD8Wd+TQHLZB4Cz82fsD3yobUxVcV5SVc9ueR1snZd7HvC3Np/5D6QzqnVICWqvOp//WeDwdtZPJRF8JH+m/8wHm+Wrtu09+WA0GHgYOCpPG0HVAavt9iAl12vy59+MdMCtbIvS2510tvAM8LW8XgcC2+dpxwN3AcPzuvtf4Mo6n33J9mozfjfSwXmlGp/pGeDDeXgQsHW9ukgH6beB/UiJbQC1E8iVeX1snrflHlXroGYCaft/V2t7kL5g/SSvo9G57t2qYltASojLAmcAd/X0saSnX76E1RqejIiLIl2LvRQYSrrcUcs+wMyI+EVELIyI+4Ffky6zVFwfEfdExELSgWR0g/HsIGke6WB6MPDJiHglT1sMnBQRb0bEv0jfBv83Iu6OiEURcSnwJunyxg6kg+w5EfF2RFwH3FtnmduRDrrfiIjXI2JBRBRtQD4EuDgi7ouIN4FvATtKGlE1z5kRMS8ingJupf46WY100KvnM6QzlBsj4m3SmeMA4INV85wbEU9HxEvA79tZ1hK5feUA4Lv5808l7QsVndnu+wDPRsQP8nqdHxF352lHAd+OiNl53Z0MHNjg5aOnAZESdFtvA5tKWiUiXo6I+zqo686I+G1ELM77Vy3/ndfRg8AvSPtop0haG9gJ+GZeR5OBnwH/XjXbHRFxQ/4/vQzYsrPLXdo5gbSGZysDEfFGHly5zrzrAttLmld5kQ6g769VH/BGO3XVc1dErBoRq0fEDhFxU9W0FyJiQZt4vtYmnrVJyWAtYE7kr3DZk3WWuTYpkS5sMFbycpbUGxGvkb6ND6uap+g6mUtK4EWXtZh0ZlBmWdWGkM4cZlWNq15Xndnua5MuUdWyLnB9VZ0Pky4X1vsCU8sw0jf5eTWmHUD61v6kpNsl7dhBXbM6mN52nidJ26Sz1gJeioj5bepub7v27+vtNE4gra9td8mzgNvzAb7yWjkivtiD8ZzWJp4VI+JK0jf5YZJUNf86deqdBaxT5x+yoy6jnyYdCAGQtBLpTGJOB+VquYl00Cu6LJEO0GWWVe0F0uXBtavGVa+rzmz3WUC922pnkdrcquvtHxGNfJ5PAvdFxOttJ0TEvRExFlgD+C3pEh3U36ZFugdvu46ezsOvAytWTatOrh3V/TQwWFL1jRLr0Pnt2qs5gbS+53j3P/8fgI0kfU5Sv/zaVtImJevrrIuAoyRtr2QlSR/P/4h3kg6Kx+U49yddqqrlHlLCOTPX0V/STlUxD5e0fJ2yVwKHSxotaQXgdODuiJhZ4vOcBHxQ0vclvR9A0gaSfiVpVdIB8OOSds+35X6NdMnuHyWWtUS+LPIb4GRJK0ralNSeVNGZ7f4HYKikL0taQdJASdvnaRcAp0laN3/WIZLGdlRh3tbDJJ1Eauw+scY8y0s6RNL78uW+V0mXQCFt09Ukva9A/G19J6+jUaQ2oavz+MnA3pIG52335Tbl6u77ETGLtA3PyPveFsARpAZ9q8MJpPX9iHRN+mVJ5+ZT7D2Bg0jfmp7lnUbtIk4GLs2XLD7d2eAiYiLpBoAfk+4im05q9CYi3gL2z+9fIrUf/KZOPYuATwAbAE+R7vb6TJ58C+mul2clvVij7E3Ad0htAs8A65PWT5nP8ziwI6mBdZqkV3K9E4H5EfEocCipof7FHPMn8mftrGNIl52eJTUI/6IqrtLbPZf9SI71WdLdaLvmyT8i3Xn0V0nzSQ3q29eqJ1tLUuXOqXtJDdm7RMRf68z/OWCmpFdJ7S2H5JgeISX+GXlfbOQy1O2k/exm4H+qln0Z6UaMmcBfeSexVJwB/Fde3tdr1Hswabs/DVxPauu7qcZ8llXu9DEzM2uIz0DMzKwUJxAzMyvFCcTMzEpxAjEzs1KW6odgVl999RgxYkRPh2FmtlSZNGnSixExpLP1LNUJZMSIEUycOLGnwzAzW6pIqtcjREN8CcvMzEpxAjEzs1KcQMzMrJSlug2klrfffpvZs2ezYMGCjmfuQ/r378/w4cPp169fT4diZr1Er0sgs2fPZuDAgYwYMYJ3dwLbd0UEc+fOZfbs2YwcObKnwzGzXqLXXcJasGABq622mpNHFUmsttpqPiszsy7V6xII4ORRg9eJmXW1XplAzMys+XpdG0hbOu+8Lq0vjj22w3lmzpzJPvvsw9SpU981/rvf/S4777wze+yxB+eccw7jx49nxRVXrFOLmVlr6/UJpJWccsopS4bPOeccDj30UCcQsz6ovS+2Rb6ktgpfwmqSRYsW8fnPf55Ro0ax55578q9//Ytx48Zx3XXXce655/L000+z6667suuuu7Jo0SLGjRvHZpttxuabb87ZZ5/d0+GbmXXICaRJHnvsMY4++mimTZvGqquuyq9//esl04477jjWWmstbr31Vm699VYmT57MnDlzmDp1Kg8++CCHH354D0ZuZlaME0iTjBw5ktGjRwOwzTbbMHPmzLrzrrfeesyYMYNjjz2WP//5z6yyyirdFKWZWXlOIE2ywgorLBledtllWbhwYd15Bw0axAMPPMAuu+zCBRdcwJFHHtkdIZqZdYob0XvIwIEDmT9/Pquvvjovvvgiyy+/PAcccAAf+MAHOPTQQ3s6PDOzDvX6BNKqdzSMHz+evfbai7XWWotzzjmHww8/nMWLFwNwxhln9HB0ZmYdU0T0dAyljRkzJtr+oNTDDz/MJpts0kMRtTavG7PW0NO38UqaFBFjOltP09pAJPWXdI+kByRNk/TfefxISXdLmi7paknL5/Er5PfT8/QRzYrNzMw6r5mN6G8Cu0XElsBoYC9JOwBnAWdHxAbAy8ARef4jgJfz+LPzfGZm1qKalkAieS2/7ZdfAewGXJfHXwrsl4fH5vfk6bvLPQCambWspt7GK2lZSZOB54EbgceBeRFRuad1NjAsDw8DZgHk6a8Aq9Woc7ykiZImvvDCC80M38zM2tHUBBIRiyJiNDAc2A7YuAvqvDAixkTEmCFDhnQ6RjMzK6dbHiSMiHnArcCOwKqSKrcPDwfm5OE5wNoAefr7gLndEZ+ZmTWuac+BSBoCvB0R8yQNAD5Cahi/FTgQuAo4DPhdLjIhv78zT78luuAe4/O6uBnl2C667XnChAk89NBDnHDCCV1Sn5lZd2vmg4RDgUslLUs607kmIv4g6SHgKkmnAvcDP8/z/xy4TNJ04CXgoCbG1uP23Xdf9t13354Ow8ystGbehTUlIraKiC0iYrOIOCWPnxER20XEBhHxqYh4M49fkN9vkKfPaFZszTZz5kw23nhjxo0bx0YbbcQhhxzCTTfdxE477cSGG27IPffcwyWXXMIxxxwDwLXXXstmm23Glltuyc477wzAtGnT2G677Rg9ejRbbLEFjz32GAC/+tWvloz/whe+wKJFi9wdvJn1iF7flUlPmT59Otdeey0XX3wx2267LVdccQV33HEHEyZM4PTTT2e//fZbMu8pp5zCX/7yF4YNG8a8efMAuOCCCzj++OM55JBDeOutt1i0aBEPP/wwV199NX//+9/p168fX/rSl7j88ssZNWrUku7ggSV1mJk1k3vjbZKRI0ey+eabs8wyyzBq1Ch23313JLH55pu/p2v3nXbaiXHjxnHRRRexaNEiAHbccUdOP/10zjrrLJ588kkGDBjAzTffzKRJk9h2220ZPXo0N998MzNmzHB38GbWI5xAmqS6O/dllllmyftlllnmPV27X3DBBZx66qnMmjWLbbbZhrlz5/LZz36WCRMmMGDAAPbee29uueUWIoLDDjuMyZMnM3nyZB599FFOPvlkdwdvZj3Cl7BawOOPP87222/P9ttvz5/+9CdmzZrFK6+8wnrrrcdxxx3HU089xZQpU9hzzz0ZO3YsX/nKV1hjjTV46aWXmD9/PiuttJK7gzezbtfrE0hX3XbbTN/4xjd47LHHiAh23313ttxyS8466ywuu+wy+vXrx/vf/35OPPFEBg8ezKmnnsqee+7J4sWL6devH+effz4DBgxwd/Bm1u3cnXsf4nVj1hrcnbuZmfVpTiBmZlZKr0wgS/NluWbxOjGzrtbrEkj//v2ZO3euD5hVIoK5c+fSv3//ng7FzHqRXncX1vDhw5k9ezb+rZB369+/P8OHD+/pMMysF+l1CaRfv36MHDmyp8MwM+v1et0lLDMz6x5OIGZmVooTiJmZleIEYmZmpTiBmJlZKU4gZmZWihOImZmV4gRiZmalOIGYmVkpTiBmZlaKE4iZmZXStAQiaW1Jt0p6SNI0Scfn8SdLmiNpcn7tXVXmW5KmS3pU0kebFZuZmXVeMztTXAh8LSLukzQQmCTpxjzt7Ij4n+qZJW0KHASMAtYCbpK0UUQsamKMZmZWUtPOQCLimYi4Lw/PBx4GhrVTZCxwVUS8GRFPANOB7ZoVn5mZdU63tIFIGgFsBdydRx0jaYqkiyUNyuOGAbOqis2mRsKRNF7SREkT/ZsfZmY9p+kJRNLKwK+BL0fEq8BPgfWB0cAzwA8aqS8iLoyIMRExZsiQIV0er5mZFdPUBCKpHyl5XB4RvwGIiOciYlFELAYu4p3LVHOAtauKD8/jzMysBTXzLiwBPwcejogfVo0fWjXbJ4GpeXgCcJCkFSSNBDYE7mlWfGZm1jnNvAtrJ+BzwIOSJudxJwIHSxoNBDAT+AJAREyTdA3wEOkOrqN9B5aZWetqWgKJiDsA1Zh0QztlTgNOa1ZMZmbWdfwkupmZleIEYmZmpXSYQCQdL2kVJT+XdJ+kPbsjODMza11FzkD+Iz+/sScwiNQwfmZTozIzs5ZXJIFUGsL3Bi6LiGnUbhw3M7M+pEgCmSTpr6QE8pfcMeLi5oZlZmatrshtvEeQuh2ZERFvSFoNOLy5YZmZWasrcgYSwKbAcfn9SkD/pkVkZmZLhSIJ5CfAjsDB+f184PymRWRmZkuFIpewto+IrSXdDxARL0tavslxmZlZiytyBvK2pGVJl7KQNAQ3opuZ9XlFEsi5wPXAGpJOA+4ATm9qVGZm1vI6vIQVEZdLmgTsTnr+Y7+IeLjpkZmZWUurm0AkDa56+zxwZfW0iHipmYGZmVlra+8MZBKp3aPWU+cBrNeUiMzMbKlQN4FExMjuDMTMzJYuhX5QStL+wIdIZx7/FxG/bWpUZmbW8op05/4T4CjgQdLvlx8lyQ8Smpn1cUXOQHYDNomIynMglwLTmhqVmZm1vCLPgUwH1ql6v3YeZ2ZmfViRM5CBwMOS7snvtwUmSpoAEBH7Nis4MzNrXUUSyHebHoWZmS11ijyJfjuApFWq5/eDhGZmfVuHCUTSeOAUYAGpE0XhBwnNzPq8Io3o3wA2i4gREbFeRIyMiA6Th6S1Jd0q6SFJ0yQdn8cPlnSjpMfy30F5vCSdK2m6pCmStu7cRzMzs2YqkkAeB94oUfdC4GsRsSmwA3C0pE2BE4CbI2JD4Ob8HuBjwIb5NR74aYllmplZNynSiP4t4B+S7gberIyMiOPqF4GIeAZ4Jg/Pl/QwMAwYC+ySZ7sUuA34Zh7/y/y8yV2SVpU0NNdjZmYtpkgC+V/gFtKT6KV+SErSCGAr4G5gzaqk8CywZh4eBsyqKjY7j3tXAsltMuMB1lmn+vEUMzPrTkUSSL+I+GrZBUhaGfg18OWIeFV6p3PfiAhJ0Uh9EXEhcCHAmDFjGiprZmZdp0gbyJ8kjZc0NDeAD27zWyF1SepHSh6XR8Rv8ujnJA3N04eSfmsEYA7pKfeK4XmcmZm1oCIJ5GByOwjpN0ImARM7KqR0qvFz4OGI+GHVpAnAYXn4MOB3VeP/Pd+NtQPwits/zMxaV5EHCcv+LshOwOeAByVNzuNOBM4ErpF0BPAk8Ok87QZgb1I/W28Ah5dcrpmZdYOivweyGbAp0L8yLiJ+2V6ZiLiD2r9mCOn31dvOH8DRReIxM7OeV+RJ9JNIt91uSjpL+BhwB9BuAjEzs96tSBvIgaQzhmcj4nBgS+B9TY3KzMxaXpEE8q+IWAwszB0qPs+775YyM7M+qEgbyERJqwIXke7Aeg24s6lRmZlZyytyF9aX8uAFkv4MrBIRU5oblpmZtbq6CUTSusC8iHglv98V2A94UtIjEfFWN8VoZmYtqL02kGuAlQAkjQauBZ4iNaL/pPmhmZlZK2vvEtaAiHg6Dx8KXBwRP5C0DDC5nXJmZtYHtHcGUv0Q4G6k3+4g35FlZmZ9XHtnILdIuobUnfogUpfulQ4Q3f5hZtbHtZdAvgx8BhgKfCgi3s7j3w98u9mBmZlZa6ubQHLfVFfVGH9/UyMyM7OlQpEn0c3MzN7DCcTMzEqpm0Ak3Zz/ntV94ZiZ2dKivUb0oZI+COwr6Sra/LZHRNzX1MjMzKyltZdAvgt8h/Tb5D9sMy1Iz4aYmVkf1d5dWNcB10n6TkR8rxtjMjOzpUCR3ni/J2lfYOc86raI+ENzwzIzs1bX4V1Yks4Ajgceyq/jJZ3e7MDMzKy1FflBqY8Doyt9YEm6FLgfOLGZgZmZWWsr+hzIqlXD/j10MzMrdAZyBnC/pFtJt/LuDJzQ1KjMzKzlFWlEv1LSbcC2edQ3I+LZpkZlZmYtr9AlrIh4JiIm5Feh5CHpYknPS5paNe5kSXMkTc6vvaumfUvSdEmPSvpo4x/FzMy6UzP7wroE2KvG+LMjYnR+3QAgaVPgIGBULvMTScs2MTYzM+ukpiWQiPgb8FLB2ccCV0XEmxHxBDAd2K5ZsZmZWee1m0AkLSvpkS5e5jGSpuRLXIPyuGHArKp5ZudxtWIaL2mipIkvvPBCF4dmZmZFtZtAImIR8KikdbpoeT8F1gdGk34q9weNVhARF0bEmIgYM2TIkC4Ky8zMGlXkNt5BwDRJ9wCvV0ZGxL6NLiwinqsMS7oIqHSJMgdYu2rW4XmcmZm1qCIJ5DtdtTBJQyPimfz2k0DlDq0JwBWSfgisBWwI3NNVyzUzs65X5DmQ2yWtC2wYETdJWhHo8A4pSVcCuwCrS5oNnATsImk0qTv4mcAX8jKmSbqG1NfWQuDofPnMzMxaVIcJRNLngfHAYFL7xTDgAmD39spFxME1Rv+8nflPA07rKB4zM2sNRW7jPRrYCXgVICIeA9ZoZlBmZtb6iiSQNyPircobScuRLkGZmVkfViSB3C7pRGCApI8A1wK/b25YZmbW6ookkBOAF4AHSY3eNwD/1cygzMys9RW5C2tx/hGpu0mXrh6NCF/CMjPr44rchfVx0l1Xj5N+D2SkpC9ExJ+aHZyZmbWuIg8S/gDYNSKmA0haH/gj4ARiZtaHFWkDmV9JHtkMYH6T4jEzs6VE3TMQSfvnwYmSbgCuIbWBfAq4txtiMzOzFtbeJaxPVA0/B/xbHn4BGNC0iMzMbKlQN4FExOHdGYiZmS1dityFNRI4FhhRPX+Z7tzNzKz3KHIX1m9JnSD+Hljc3HDMzGxpUSSBLIiIc5seiZmZLVWKJJAfSToJ+CvwZmVkRNzXtKjMzKzlFUkgmwOfA3bjnUtYkd+bmVkfVSSBfApYr7pLdzMzsyJPok8FVm12IGZmtnQpcgayKvCIpHt5dxuIb+M1M+vDiiSQk5oehZmZLXWK/B7I7d0RiJmZLV2KPIk+n3d+A315oB/wekSs0szAzMystRU5AxlYGZYkYCywQzODMjOz1lfkLqwlIvkt8NGO5pV0saTnJU2tGjdY0o2SHst/B+XxknSupOmSpkjauuFPYmZm3arDBCJp/6rXgZLOBBYUqPsSYK82404Abo6IDYGb83uAjwEb5td44KcF4zczsx5S5C6s6t8FWQjMJF3GaldE/E3SiDajxwK75OFLgduAb+bxv4yIAO6StKqkoRHxTIH4zMysBxRpA+nK3wVZsyopPAusmYeHAbOq5pudx70ngUgaTzpLYZ111unC0MzMrBHt/aTtd9spFxHxvc4sOCJCUnQ853vKXQhcCDBmzJiGy5uZ9QY677y60+LYY7slhvbaQF6v8QI4gnTZqYznJA0FyH+fz+PnAGtXzTc8jzMzsxZVN4FExA8qL9I3/gHA4cBVwHollzcBOCwPHwb8rmr8v+e7sXYAXnH7h5lZa2u3DUTSYOCrwCGkRu+tI+LlIhVLupLUYL66pNmkLlHOBK6RdATwJPDpPPsNwN7AdOANUqIyM7MW1l4byPeB/UlnH5tHxGuNVBwRB9eZtHuNeQM4upH6zcysZ7XXBvI1YC3gv4CnJb2aX/Mlvdo94ZmZWauqewYSEQ09pW5mZn2Lk4SZmZXiBGJmZqUU6crEzMyyVniAr1X4DMTMzEpxAjEzs1KcQMzMrBQnEDMzK8UJxMzMSnECMTOzUpxAzMysFCcQMzMrxQnEzMxKcQIxM7NSnEDMzKwUJxAzMyvFCcTMzEpxAjEzs1KcQMzMrBQnEDMzK8UJxMzMSnECMTOzUpxAzMyslB75TXRJM4H5wCJgYUSMkTQYuBoYAcwEPh0RL/dEfGZm1rGePAPZNSJGR8SY/P4E4OaI2BC4Ob83M7MW1UqXsMYCl+bhS4H9ejAWMzPrQE8lkAD+KmmSpPF53JoR8UwefhZYs1ZBSeMlTZQ08YUXXuiOWM3MrIYeaQMBPhQRcyStAdwo6ZHqiRERkqJWwYi4ELgQYMyYMTXnMTOz5uuRM5CImJP/Pg9cD2wHPCdpKED++3xPxGZmZsV0ewKRtJKkgZVhYE9gKjABOCzPdhjwu+6OzczMiuuJS1hrAtdLqiz/ioj4s6R7gWskHQE8CXy6B2IzM7OCuj2BRMQMYMsa4+cCu3d3PGZmVk4r3cZrZmZLEScQMzMrxQnEzMxKcQIxM7NSnEDMzKwUJxAzMyvFCcTMzEpxAjEzs1KcQMzMrBQnEDMzK8UJxMzMSnECMTOzUpxAzMysFCcQMzMrxQnEzMxKcQIxM7NSnEDMzKwUJxAzMyvFCcTMzEpxAjEzs1KcQMzMrBQnEDMzK2W5ng7AzKwonXde3Wlx7LGdqqNoeXuHE4iZ9QrnSTXHHxvRzZH0HS2XQCTtBfwIWBb4WUSc2cMhmZkVUi+JQe9MZC2VQCQtC5wPfASYDdwraUJEPNSzkZn1Tn3tgGddq6USCLAdMD0iZgBIugoYCziBmHVCvev+53ZzHK3MybRxihZaMZIOBPaKiCPz+88B20fEMVXzjAfG57cfAB7t5GJXB17s4TpaIYZWqaMVYmiVOlohhlapoxViaJU6uiKGD0TEwE7W0XJnIB2KiAuBC7uqPkkTI2JMT9bRCjG0Sh2tEEOr1NEKMbRKHa0QQ6vU0VUxdKZ8Ras9BzIHWLvq/fA8zszMWkyrJZB7gQ0ljZS0PHAQMKGHYzIzsxpa6hJWRCyUdAzwF9JtvBdHxLQmL7YrLod1to5WiKFV6miFGFqljlaIoVXqaIUYWqWOVogBaLFGdDMzW3q02iUsMzNbSjiBmJlZORHRa1/AXqTnRKYDJ9SYvgJwdZ5+NzAijx8B/At4HFgAvFKn/M7AfcBC4MA20w4DHiPdRfZMOzG0V8ciYHKO47V26vgq6WHLKcDNwLpt4pgDvAU8V6J80RiOAh7M894BbFo17Vu53Kz8aqiOqu1RiWNevTqq6joACGBMmziezutiTiPlG4kBGAe8kOedDBxZYr9or45C2yTP++m8bacBVzSyX3RQvuh+cXbVZ/gnMK/EumivjqJxrAPcCtxP2s/3bmS/qFe+wf1iXdL/1xTgNmB4iXXRXh2LgLnA28CrdbanSM+PTs91bF0jhseAw+r9b7w52Qd3AAAI2UlEQVSrviIzLY0vUiP848B6wPLAA1Qd1PI8XwIuyMMHAVdX7RRTC5QfAWwB/JKqgz8wGJhBeuDnCeApYI1G6sjTXiv4OXYFVszDX6z6HJU4ngBG5+GpRcs3GMMqVcP7An/Ow5vm+QcAT+ZX/wbrqGyPDuPI8w8E/gbcxTsJoBLH46Sk/XiD5QvHQDr4/7hGvY3sFzXraHCbbEg64A3K79docL+oWb6RGNrUdyzpxpiG1kW9OhpcFxcCX6zaF2Y2sl+0U76R/eJa8oEZ2A24rMR+UbOOqnWxM7A1MLXO+tsb+BMpkewA3N0mhsHAoDw8qKPjbG++hLWkW5SIeAuodItSbSxwaR6+DthdWtKfwYCOykfEzIiYAixuU+9HgRtJ/4D/JG2w3Ruso/DniIhbI+KN/PYu0vMzlTgeAP4ZEZNzTA82UL6RGF6tersS6ds7eb6rSAeqR/JrqwbrKBxH9j3gLNLZY8VY4B+5/N9I38D+0UD5RmOopfB+UUCROD4PnB8RLwNExPNVcXS4X7RTvpEYqh0MXFkVQ5l1UV1HI3EEsEoefh/pjAOK7xf1yjcSw6bALXn41qrpjayLenWkINNneIn6xgK/jOQuYFVJQysxRMRLeXvfSLqC067enECGkS6XVMzO42rOExELSZeqVquato2k2yV9uE75jpZd+Vsp20gdkL6p/wrYStJ+7XyOakeQdsBKHAt4Zz3MJv0jFC3fUAySjpb0OPD/gOOqYii8LurUATASuALYIm+PmnFI2hpYOyL+2KbqYaRLFNXr4u0GyheOITtA0hRJ10mqPBzb6H5Rqw4ovk02AjaS9HdJd+WeritxFNkv6pVvJAYAJK1LWn+Vg1/D/yM16mgkjpOBQyXNBm4gnclU4uhwv2inPBTfLx4A9s/DnwQGSlqNxtZFvToA+ucnzK8nnUXXUu+4WOR4+R69OYF0xjOk6/G/JbUPXEHaUbvbusA3Sd80zpG0fnszSzoUGAN8v8zC6pQvHENEnB8R6+f5/6tMDHXqeIZ0DfobpDOkKySt0raspGWAHwJfK7PsDsoXiiH7Pak9bQvSN7lL68zXnvbqKLpNliN9q92F9M39IkmrNhBDe+Ub2jdJl4ivi4hFDSy/SB1F4zgYuCQihpMu41yWt3dR9co3sl98Hfg3SfcD/0Zq72h0fbRXx7qRujg5DhhaYJt0Wm9OIEW6RVkyj6TlSKemcyPiTVLj+9oRMYl0bXPLGuU7Wnblb2XZDXXNEhFz8vyrkhrMtqpXh6Q9gG8D++b4K3H05531MJx07bNo+YZiqHIVUPk2WHZdLKkjIt6MiLl5/hVJ22OjGnUMBDYDbpM0k3SNd4KkMXm+5dusi35FyzcQAxExt2od/gzYptF10U4djWyT2cCEiHg7Ip4gXR7ZkOL7Rb3yZfaLg3j3pacy+0XbOhqJ4wjgmlzmzvz5V6fYflG3fIP7xdMRsX9EbEX6XyMi5jWyLtqpo7IuIJ1JvJ7XRVv1jovlupHqqJFkaX2Rvj3NIJ1eVhq1RrWZ52je3Yh+TR4eksvMIDVKzSE1lI2qs6xLeG8j+hOkHXRm3qBr1oqhnToGke4SWy7X8QQpidX6HFuRdtwN24yvxDEzl32i1udop3wjMWxYNfwJYGIeHpXnX5HUOPgU7zSiF61jCKmRcrlc/tmO1mcudxvvNIJX4pgBfDj/baR84RiAoVXDnwTuanS/aKeORrbJXsCleXj1vLzVGtgv6pUvHEMuu3GeT2X/R+rU0ci6+BMwLg9vQmrDUNH9op3yjewXqwPL5OHTgFNK7Bf16hgErFD1//wmtW8Q+TjvbkS/p00Mg/LrCWBwh8fZjmZYml+kU81/kg6O387jTiF9y4Z0ILuW1HB2D7BeHn8A6bbFym28z9Qpvy3pW9rrpNvnplUt+z9yvU/nnapeDDXrAD5Iath8IO9U7dVxE+lWzMqtjhPaxFG5RfH5Rso3GMOP8jqbTLqcMKoqhm/ncrPzq6E6qrbH5LxO59Sro832v41338b77bwt38rrpHD5RmIAzsjzPpA/x8Yl9ouadTS4TUS6JPdQLnNQg/tFzfKNxJDfnwycWWP9FloX9epocF1sCvw9zzsZ2LOR/aJeeRrbLw4k3SL7T9JZ5Qol9ouadVSti5dJbTgLSf9rR5Auxx9VtU3Pz/U/yLv/PyoxTAcOL3KMdVcmZmZWSm9uAzEzsyZyAjEzs1KcQMzMrBQnEDMzK8UJxMzMSnECsT5F0vslXSXpcUmTJN0gaaMS9XxY0jRJkyUNk3Rdnfluyw8ymvU6TiDWZ+SOMq8HbouI9SNiG1JX3muWqO4Q4IyIGB0RcyLiwK6M1Wxp4ARifcmuwNsRcUFlREQ8ANwh6fuSpkp6UNJnACTtks8grpP0iKTLlRxJ+p2M7+VxIyRNzWUG5DOchyVdT+rVmTxtT0l3SrpP0rWSVs7jZ0r67zz+QUkb5/ErS/pFHjdF0gHt1WPW3ZxArC/ZDJhUY/z+pO7mtwT2AL6fu7iG1C3El0lPIq8H7BQRPwMmAN+IiEPa1PVF4I2I2AQ4idyHlaTVSZ1D7hERWwMTSR11VryYx/+U1GEewHeAVyJi80idKt5SoB6zbrNcTwdg1gI+BFwZqZfX5yTdTupi5lVSX0GzASRNJv2A0B3t1LUz6RffiIgpkqbk8TuQu8PIPzmzPHBnVbnf5L+TeKe77j1IfbSR63tZ0j4d1GPWbZxArC+ZRupLqBFvVg0vovz/jEg/2HNwB8vpaBkd1WPWbXwJy/qSW4AVJI2vjJC0Bel3rD8jaVlJQ0hnEfeUXMbfgM/mujcj/VwxpN+K2EnSBnnaSgXu/rqR1GN0JdZBJesxawonEOszIvUc+klgj3wb7zRSr7dXAFNIPa3eAvxnRDxbcjE/BVaW9DCpF9VJedkvkH7n/Mp8WetOUhfl7TkVGJQb9x8Adi1Zj1lTuDdeMzMrxWcgZmZWihOImZmV4gRiZmalOIGYmVkpTiBmZlaKE4iZmZXiBGJmZqX8fx9m7owAo/CgAAAAAElFTkSuQmCC\n", - "text/plain": [ - "
" - ] - }, - "metadata": { - "needs_background": "light" - }, - "output_type": "display_data" - }, - { - "name": "stderr", + "name": "stdout", "output_type": "stream", "text": [ - "INFO:rasa_nlu.evaluate:Entity evaluation results:\n", - "INFO:rasa_nlu.evaluate:Evaluation for entity extractor: ner_crf \n", - "INFO:rasa_nlu.evaluate:F1-Score: 0.9907693995778069\n", - "INFO:rasa_nlu.evaluate:Precision: 0.9915729577474618\n", - "INFO:rasa_nlu.evaluate:Accuracy: 0.9909584086799277\n", - "INFO:rasa_nlu.evaluate:Classification report: \n", - " precision recall f1-score support\n", - "\n", - " bff 1.00 1.00 1.00 17\n", - " book 1.00 1.00 1.00 5\n", - " color 1.00 1.00 1.00 13\n", - " creators 1.00 0.86 0.92 7\n", - " food 1.00 1.00 1.00 17\n", - " genre 0.83 1.00 0.91 15\n", - " hobby 1.00 1.00 1.00 11\n", - " how 1.00 1.00 1.00 24\n", - " joke 1.00 1.00 1.00 7\n", - " languages 1.00 0.93 0.96 14\n", - " license 1.00 0.86 0.92 14\n", - " live 1.00 1.00 1.00 13\n", - " me 1.00 0.86 0.92 7\n", - " movie 1.00 1.00 1.00 4\n", - " name 1.00 1.00 1.00 4\n", - " no_entity 0.99 1.00 0.99 1379\n", - " ok 0.93 0.93 0.93 15\n", - " playlist 1.00 1.00 1.00 7\n", - "relationship 1.00 1.00 1.00 9\n", - " religion 1.00 1.00 1.00 16\n", - " sad 1.00 1.00 1.00 7\n", - " sign 0.50 0.67 0.57 3\n", - " sons 1.00 1.00 1.00 11\n", - " sport 1.00 1.00 1.00 6\n", - " starwars 1.00 1.00 1.00 13\n", - " story 1.00 1.00 1.00 6\n", - " towel 1.00 1.00 1.00 4\n", - " weather 1.00 1.00 1.00 8\n", - " where 1.00 0.33 0.50 3\n", - "\n", - " avg / total 0.99 0.99 0.99 1659\n", - "\n" + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - F1-Score: 1.0\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Precision: 1.0\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Accuracy: 1.0\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Classification report: \r\n", + " precision recall f1-score support\r\n", + "\r\n", + " bff 1.00 1.00 1.00 17\r\n", + " comida 1.00 1.00 1.00 17\r\n", + " cor 1.00 1.00 1.00 13\r\n", + " esporte 1.00 1.00 1.00 6\r\n", + " filhos 1.00 1.00 1.00 11\r\n", + " filme 1.00 1.00 1.00 4\r\n", + " genero 1.00 1.00 1.00 15\r\n", + " historia 1.00 1.00 1.00 6\r\n", + " hobby 1.00 1.00 1.00 11\r\n", + " how 1.00 1.00 1.00 24\r\n", + " license 1.00 1.00 1.00 14\r\n", + " linguagens 1.00 1.00 1.00 14\r\n", + " live 1.00 1.00 1.00 13\r\n", + " me 1.00 1.00 1.00 6\r\n", + " no_entity 1.00 1.00 1.00 613\r\n", + " piada 1.00 1.00 1.00 7\r\n", + " playlist 1.00 1.00 1.00 7\r\n", + "relationship 1.00 1.00 1.00 9\r\n", + " religiao 1.00 1.00 1.00 16\r\n", + " signo 1.00 1.00 1.00 3\r\n", + " starwars 1.00 1.00 1.00 13\r\n", + " triste 1.00 1.00 1.00 7\r\n", + " where 1.00 1.00 1.00 3\r\n", + "\r\n", + " micro avg 1.00 1.00 1.00 849\r\n", + " macro avg 1.00 1.00 1.00 849\r\n", + "weighted avg 1.00 1.00 1.00 849\r\n", + "\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished evaluation\r\n" ] } ], "source": [ - "from rasa_nlu.evaluate import run_evaluation\n", - "run_evaluation('../../bot/data/intents/', model_directory)" + "!python -m rasa_nlu.test -d $COACH_INTENTS_PATH -m $COACH_MODELS_NLU_PATH --mode evaluation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### Lista de Problemas\n", - "\n", - "O arquivo `erros,json` mostra os erros encontrados após executar o comando `rasa_nlu.evaluate.run_evaluation`.\n", - "\n", - "Normalmente os erros mostrados são textos repetidos nos exemplos de diferrentes `intents`.\n", - "\n", - "Caso o arquivo não seja gerado significa que não foram encontrados erros." + "#### Matriz de Confusão\n", + "* A matriz de confusão mostra a correlação entre as intents.\n", + "* A diagonal principal tem forte correlação pois mostra a relação de uma intent **com ela mesma**\n", + "* O ideal é que não haja **nenhum valor** diferente de **0 fora da diagonal principal**." ] }, { "cell_type": "code", "execution_count": 5, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "IFrame(src='./confmat.png', width=900, height=700)" + ] + }, + { + "cell_type": "markdown", "metadata": {}, + "source": [ + "#### Erros\n", + "* O arquivo `erros,json` mostra os erros encontrados. Este arquivo lista os mesmos erros mostrados na **matriz de confusão**, então caso ela fique muito grande, não se preocupe, pois você pode apenas procurar os erros no arquivo gerado.\n", + "* Os erros mostrados são textos repetidos nos exemplos de diferrentes `intents`.\n", + "* Caso o arquivo não seja gerado significa que não foram encontrados erros." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{\r\n", - " \"intent_errors\": [\r\n", - " {\r\n", - " \"text\": \"software livre\",\r\n", - " \"intent\": \"license\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"software_livre\",\r\n", - " \"confidence\": 0.8758267164230347\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"e o c#\",\r\n", - " \"intent\": \"languages\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"\",\r\n", - " \"confidence\": 0.0\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"Tudo bem?\",\r\n", - " \"intent\": \"its_ok\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"afirmar\",\r\n", - " \"confidence\": 0.6227023005485535\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"onde voce nasceu?\",\r\n", - " \"intent\": \"where_r_u_from\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"quem_eh_voce\",\r\n", - " \"confidence\": 0.7670958042144775\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"Quem te fez?\",\r\n", - " \"intent\": \"creators\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"o_que_eh_lappis\",\r\n", - " \"confidence\": 0.8802075982093811\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"Qual o seu signo?\",\r\n", - " \"intent\": \"sign\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"meu_signo\",\r\n", - " \"confidence\": 0.7709224820137024\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"chatbot\",\r\n", - " \"intent\": \"quem_eh_lappisudo\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"quem_eh_voce\",\r\n", - " \"confidence\": 0.8417193293571472\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"signo\",\r\n", - " \"intent\": \"meu_signo\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"sign\",\r\n", - " \"confidence\": 0.7748193740844727\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"java\",\r\n", - " \"intent\": \"zoacao_java\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"languages\",\r\n", - " \"confidence\": 0.7685664892196655\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"como vai\",\r\n", - " \"intent\": \"cumprimentar\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"tudo_bem\",\r\n", - " \"confidence\": 0.7766595482826233\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"tudo bom\",\r\n", - " \"intent\": \"cumprimentar\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"tudo_bem\",\r\n", - " \"confidence\": 0.777603030204773\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"Tudo bem\",\r\n", - " \"intent\": \"tudo_bem\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"afirmar\",\r\n", - " \"confidence\": 0.6227023005485535\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"Quem te criou?\",\r\n", - " \"intent\": \"quem_eh_voce\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"o_que_eh_lappis\",\r\n", - " \"confidence\": 0.8640211820602417\r\n", - " }\r\n", - " },\r\n", - " {\r\n", - " \"text\": \"Quem te desenvolveu?\",\r\n", - " \"intent\": \"quem_eh_voce\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"o_que_eh_lappis\",\r\n", - " \"confidence\": 0.8942159414291382\r\n", - " }\r\n", + "[\r\n", + " {\r\n", + " \"text\": \"como vai\",\r\n", + " \"intent\": \"cumprimentar\",\r\n", + " \"intent_prediction\": {\r\n", + " \"name\": \"tudo_bem\",\r\n", + " \"confidence\": 0.7145657539367676\r\n", + " }\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"tudo bom\",\r\n", + " \"intent\": \"cumprimentar\",\r\n", + " \"intent_prediction\": {\r\n", + " \"name\": \"tudo_bem\",\r\n", + " \"confidence\": 0.7155274152755737\r\n", + " }\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"e o c#\",\r\n", + " \"intent\": \"linguagens\",\r\n", + " \"intent_prediction\": {\r\n", + " \"name\": \"\",\r\n", + " \"confidence\": 0.0\r\n", " }\r\n", - " ]\r\n", - "}" + " }\r\n", + "]" ] } ], @@ -551,86 +497,254 @@ }, { "cell_type": "markdown", - "metadata": { - "colab_type": "text", - "id": "1xeXgpdwzOAl" - }, + "metadata": {}, "source": [ - "### Avaliação do modelo de NLU treinado\n", + "#### Histograma\n", "\n", - "Caso queira testar mensagens específicas mande a mensagem na célula seguinte:" + "* O histograma contém a distribuição da predições das `intents`" ] }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, + "metadata": { + "scrolled": false + }, + "outputs": [ + { + "data": { + "text/html": [ + "\n", + " \n", + " " + ], + "text/plain": [ + "" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "IFrame(src='./hist.png', width=900, height=700)" + ] + }, + { + "cell_type": "markdown", "metadata": {}, + "source": [ + "### Crossvalidation" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "scrolled": true + }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "{\n", - " \"intent\": {\n", - " \"name\": \"quem_eh_voce\",\n", - " \"confidence\": 0.8361726403236389\n", - " },\n", - " \"entities\": [],\n", - " \"intent_ranking\": [\n", - " {\n", - " \"name\": \"quem_eh_voce\",\n", - " \"confidence\": 0.8361726403236389\n", - " },\n", - " {\n", - " \"name\": \"where_r_u_from\",\n", - " \"confidence\": 0.3619411289691925\n", - " },\n", - " {\n", - " \"name\": \"despedir\",\n", - " \"confidence\": 0.3474047780036926\n", - " },\n", - " {\n", - " \"name\": \"genre\",\n", - " \"confidence\": 0.2647324502468109\n", - " },\n", - " {\n", - " \"name\": \"sign\",\n", - " \"confidence\": 0.2186793088912964\n", - " },\n", - " {\n", - " \"name\": \"cumprimentar\",\n", - " \"confidence\": 0.20706906914710999\n", - " },\n", - " {\n", - " \"name\": \"meu_signo\",\n", - " \"confidence\": 0.19841191172599792\n", - " },\n", - " {\n", - " \"name\": \"story\",\n", - " \"confidence\": 0.18699833750724792\n", - " },\n", - " {\n", - " \"name\": \"laugh\",\n", - " \"confidence\": 0.1862882375717163\n", - " },\n", - " {\n", - " \"name\": \"erro_resposta_utter\",\n", - " \"confidence\": 0.15251879394054413\n", - " }\n", - " ],\n", - " \"text\": \"O que \\u00e9 um bot?\"\n", - "}\n" + "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\n", + " warn(RuntimeWarning(msg))\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/geral.md is md\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'cumprimentar', 'tudo_bem', 'negar', 'elogios', 'despedir', 'out_of_scope', 'diga_mais'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/aleatorio.md is md\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 197 (24 distinct intents)\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'comida', 'cor', 'historia', 'triste', 'como_estou', 'filme', 'star_wars', 'relationship', 'esporte', 'signo', 'me', 'linguagens', 'de_onde_voce_eh', 'religiao', 'hobby', 'playlist', 'license', 'onde_voce_mora'\n", + "\t- entity examples: 172 (22 distinct entities)\n", + "\t- found entities: 'live', 'filhos', 'genero', 'relationship', 'linguagens', 'me', 'hobby', 'starwars', 'playlist', 'license', 'bff', 'piada', 'comida', 'cor', 'historia', 'triste', 'where', 'filme', 'how', 'esporte', 'signo', 'religiao'\n", + "\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/actions.md is md\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 6 (1 distinct intents)\n", + "\t- Found intents: 'action_test'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 288 (32 distinct intents)\n", + "\t- Found intents: 'filhos', 'genero', 'time', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'tudo_bem', 'linguagens', 'me', 'hobby', 'playlist', 'license', 'onde_voce_mora', 'bff', 'risada', 'piada', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'elogios', 'filme', 'action_test', 'esporte', 'signo', 'de_onde_voce_eh', 'despedir', 'religiao', 'diga_mais'\n", + "\t- entity examples: 172 (22 distinct entities)\n", + "\t- found entities: 'live', 'filhos', 'genero', 'relationship', 'linguagens', 'me', 'hobby', 'starwars', 'playlist', 'license', 'bff', 'piada', 'comida', 'cor', 'historia', 'triste', 'where', 'filme', 'how', 'esporte', 'signo', 'religiao'\n", + "\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 273 (28 distinct intents)\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 162 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", + "\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 131 (28 distinct intents)\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 79 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", + "\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 142 (28 distinct intents)\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 83 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", + "\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "\n", + "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", + "For more information, please see:\n", + " * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n", + " * https://github.com/tensorflow/addons\n", + "If you depend on functionality not listed there, please file an issue.\n", + "\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use keras.layers.dense instead.\u001b[0m\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Colocations handled automatically by placer.\u001b[0m\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use keras.layers.dropout instead.\u001b[0m\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", + "2019-07-17 16:32:22 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Use tf.cast instead.\u001b[0m\n", + "2019-07-17 16:32:22 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "Instructions for updating:\n", + "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2019-07-17 16:32:22.678227: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:32:22.700933: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:32:22.701522: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55be5a7c66c0 executing computations on platform Host. Devices:\n", + "2019-07-17 16:32:22.701579: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:32:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:05<00:00, 53.69it/s, loss=0.194, acc=0.992]\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.194, train accuracy=0.992\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 131/131 [00:00<00:00, 435.20it/s]\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 142/142 [00:00<00:00, 527.33it/s]\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 142 (28 distinct intents)\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 83 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", + "\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 131 (28 distinct intents)\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 79 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", + "\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "2019-07-17 16:32:30 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:06<00:00, 45.43it/s, loss=0.188, acc=1.000]\n", + "2019-07-17 16:32:36 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.188, train accuracy=1.000\n", + "2019-07-17 16:32:36 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:36 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 142/142 [00:00<00:00, 433.04it/s]\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 131/131 [00:00<00:00, 569.47it/s]\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - CV evaluation (n=2)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent evaluation results\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Accuracy: 0.993 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train F1-score: 0.994 (0.002)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Precision: 0.997 (0.003)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Accuracy: 0.536 (0.029)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test F1-score: 0.561 (0.031)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Precision: 0.681 (0.027)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity evaluation results\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity extractor: CRFEntityExtractor\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Accuracy: 1.000 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train F1-score: 1.000 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Precision: 1.000 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity extractor: CRFEntityExtractor\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Accuracy: 0.826 (0.004)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test F1-score: 0.776 (0.001)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Precision: 0.810 (0.033)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished evaluation\n" ] } ], "source": [ - "pprint(interpreter.parse('O que é um bot?'))" + "!python -m rasa_nlu.test -d $COACH_INTENTS_PATH -c $COACH_NLU_CONFIG_PATH --mode crossvalidation --folds 2 --report" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Referências:\n", + "\n", + "O Rasa está em constante evolução, alguns links úteis para a construção deste jupyter-notebook e para a análise das `intents` são:\n", + "\n", + "* [Evaluation](https://rasa.com/docs/nlu/evaluation/)" ] }, { "cell_type": "code", "execution_count": null, - "metadata": {}, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [] } @@ -670,7 +784,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.9" } }, "nbformat": 4, diff --git a/notebooks/intents/intents-analysis.ipynb b/notebooks/intents/intents-analysis.ipynb index ca6900e0..c15cddbf 100644 --- a/notebooks/intents/intents-analysis.ipynb +++ b/notebooks/intents/intents-analysis.ipynb @@ -32,14 +32,6 @@ "text": [ "rasa_nlu: 0.15.0\n" ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n" - ] } ], "source": [ @@ -67,56 +59,54 @@ "cell_type": "code", "execution_count": 2, "metadata": { - "scrolled": true + "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "make: Entering directory '/work/bot'\n", + "make: Entering directory '/work/coach'\n", "python3 -m rasa_nlu.train -c nlu_config.yml --fixed_model_name current \\\n", - " --data data/intents/ -o models --project nlu --verbose\n", - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", + "--data data/intents/ -o /src_models --project nlu --verbose\n", "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.train' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.train'; this may result in unpredictable behaviour\n", " warn(RuntimeWarning(msg))\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/aleatorio.md is md\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/geral.md is md\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'elogios', 'negar', 'despedir', 'diga_mais', 'tudo_bem', 'out_of_scope', 'cumprimentar'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/aleatorio.md is md\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 197 (24 distinct intents)\n", - "\t- Found intents: 'historia', 'de_onde_voce_eh', 'triste', 'filhos', 'hobby', 'signo', 'como_estou', 'piada', 'playlist', 'time', 'genero', 'relationship', 'bff', 'esporte', 'risada', 'star_wars', 'religiao', 'cor', 'linguagens', 'filme', 'me', 'onde_voce_mora', 'comida', 'license'\n", + "\t- Found intents: 'bff', 'filhos', 'license', 'de_onde_voce_eh', 'triste', 'religiao', 'playlist', 'signo', 'risada', 'time', 'comida', 'star_wars', 'cor', 'onde_voce_mora', 'esporte', 'piada', 'me', 'historia', 'linguagens', 'filme', 'hobby', 'relationship', 'como_estou', 'genero'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'historia', 'filhos', 'hobby', 'playlist', 'genero', 'esporte', 'linguagens', 'filme', 'me', 'where', 'comida', 'triste', 'signo', 'starwars', 'piada', 'how', 'relationship', 'bff', 'live', 'religiao', 'cor', 'license'\n", + "\t- found entities: 'filhos', 'starwars', 'license', 'triste', 'live', 'religiao', 'signo', 'cor', 'how', 'esporte', 'piada', 'me', 'historia', 'hobby', 'relationship', 'bff', 'playlist', 'comida', 'filme', 'linguagens', 'genero', 'where'\n", "\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/actions.md is md\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/actions.md is md\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 6 (1 distinct intents)\n", "\t- Found intents: 'action_test'\n", "\t- entity examples: 0 (0 distinct entities)\n", "\t- found entities: \n", "\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/geral.md is md\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 85 (7 distinct intents)\n", - "\t- Found intents: 'elogios', 'out_of_scope', 'negar', 'despedir', 'diga_mais', 'cumprimentar', 'tudo_bem'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 288 (32 distinct intents)\n", - "\t- Found intents: 'historia', 'de_onde_voce_eh', 'out_of_scope', 'filhos', 'hobby', 'playlist', 'diga_mais', 'genero', 'esporte', 'star_wars', 'negar', 'linguagens', 'filme', 'me', 'comida', 'elogios', 'triste', 'signo', 'como_estou', 'piada', 'time', 'relationship', 'tudo_bem', 'bff', 'risada', 'action_test', 'religiao', 'despedir', 'cor', 'onde_voce_mora', 'cumprimentar', 'license'\n", + "\t- Found intents: 'filhos', 'license', 'negar', 'action_test', 'triste', 'religiao', 'signo', 'time', 'star_wars', 'cor', 'elogios', 'esporte', 'piada', 'me', 'historia', 'hobby', 'relationship', 'tudo_bem', 'como_estou', 'cumprimentar', 'bff', 'despedir', 'de_onde_voce_eh', 'playlist', 'risada', 'comida', 'onde_voce_mora', 'linguagens', 'filme', 'diga_mais', 'out_of_scope', 'genero'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'historia', 'filhos', 'hobby', 'playlist', 'genero', 'esporte', 'linguagens', 'filme', 'me', 'where', 'comida', 'triste', 'signo', 'starwars', 'piada', 'how', 'relationship', 'bff', 'live', 'religiao', 'cor', 'license'\n", + "\t- found entities: 'filhos', 'starwars', 'license', 'triste', 'live', 'religiao', 'signo', 'cor', 'how', 'esporte', 'piada', 'me', 'historia', 'hobby', 'relationship', 'bff', 'playlist', 'comida', 'filme', 'linguagens', 'genero', 'where'\n", "\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:31 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", - "2019-05-03 21:20:32 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:32 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:02 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:30:03 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:03 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", "For more information, please see:\n", @@ -124,40 +114,40 @@ " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n", - "2019-05-03 21:20:32 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use keras.layers.dense instead.\u001b[0m\n", - "2019-05-03 21:20:32 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\u001b[0m\n", - "2019-05-03 21:20:32 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use keras.layers.dropout instead.\u001b[0m\n", - "2019-05-03 21:20:32 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", - "2019-05-03 21:20:32 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use tf.cast instead.\u001b[0m\n", - "2019-05-03 21:20:32 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:30:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n", - "2019-05-03 21:20:33.153976: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-05-03 21:20:33.173206: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2905000000 Hz\n", - "2019-05-03 21:20:33.173981: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x559bc71337e0 executing computations on platform Host. Devices:\n", - "2019-05-03 21:20:33.174055: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-05-03 21:20:33 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:09<00:00, 30.99it/s, loss=0.115, acc=0.993]\n", - "2019-05-03 21:20:42 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.115, train accuracy=0.993\n", - "2019-05-03 21:20:42 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:43 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Successfully saved model into '/work/bot/models/nlu/current'\n", - "2019-05-03 21:20:43 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished training\n", - "make: Leaving directory '/work/bot'\n" + "2019-07-17 16:30:06.320809: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:30:06.344961: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:30:06.345414: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x563c2cae7710 executing computations on platform Host. Devices:\n", + "2019-07-17 16:30:06.345467: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:30:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:14<00:00, 25.65it/s, loss=0.101, acc=0.993]\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.101, train accuracy=0.993\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Successfully saved model into '/src_models/nlu/current'\n", + "2019-07-17 16:30:20 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished training\n", + "make: Leaving directory '/work/coach'\n" ] } ], "source": [ - "!make train-nlu -C $BOT_DIR_PATH" + "!make train-nlu -C $COACH_DIR_PATH" ] }, { @@ -191,38 +181,36 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", - "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\n", - " warn(RuntimeWarning(msg))\n", - "usage: test.py [-h] [--debug] [-v] -d DATA [--mode MODE] [-c CONFIG]\n", - " [-m MODEL] [-f FOLDS] [--report [REPORT]]\n", - " [--successes [SUCCESSES]] [--errors ERRORS]\n", - " [--histogram HISTOGRAM] [--confmat CONFMAT]\n", - "\n", - "evaluate a Rasa NLU pipeline with cross validation or on external data\n", - "\n", - "optional arguments:\n", - " -h, --help show this help message and exit\n", - " --debug Print lots of debugging statements. Sets logging level\n", - " to DEBUG\n", - " -v, --verbose Be verbose. Sets logging level to INFO\n", - " -d DATA, --data DATA file containing training/evaluation data\n", - " --mode MODE evaluation|crossvalidation (evaluate pretrained model\n", - " or train model by crossvalidation)\n", - " -c CONFIG, --config CONFIG\n", - " model configuration file (crossvalidation only)\n", - " -m MODEL, --model MODEL\n", - " path to model (evaluation only)\n", - " -f FOLDS, --folds FOLDS\n", - " number of CV folds (crossvalidation only)\n", - " --report [REPORT] output path to save the intent/entitymetrics report\n", - " --successes [SUCCESSES]\n", - " output path to save successful predictions\n", - " --errors ERRORS output path to save model errors\n", - " --histogram HISTOGRAM\n", - " output path for the confidence histogram\n", - " --confmat CONFMAT output path for the confusion matrix plot\n" + "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\r\n", + " warn(RuntimeWarning(msg))\r\n", + "usage: test.py [-h] [--debug] [-v] -d DATA [--mode MODE] [-c CONFIG]\r\n", + " [-m MODEL] [-f FOLDS] [--report [REPORT]]\r\n", + " [--successes [SUCCESSES]] [--errors ERRORS]\r\n", + " [--histogram HISTOGRAM] [--confmat CONFMAT]\r\n", + "\r\n", + "evaluate a Rasa NLU pipeline with cross validation or on external data\r\n", + "\r\n", + "optional arguments:\r\n", + " -h, --help show this help message and exit\r\n", + " --debug Print lots of debugging statements. Sets logging level\r\n", + " to DEBUG\r\n", + " -v, --verbose Be verbose. Sets logging level to INFO\r\n", + " -d DATA, --data DATA file containing training/evaluation data\r\n", + " --mode MODE evaluation|crossvalidation (evaluate pretrained model\r\n", + " or train model by crossvalidation)\r\n", + " -c CONFIG, --config CONFIG\r\n", + " model configuration file (crossvalidation only)\r\n", + " -m MODEL, --model MODEL\r\n", + " path to model (evaluation only)\r\n", + " -f FOLDS, --folds FOLDS\r\n", + " number of CV folds (crossvalidation only)\r\n", + " --report [REPORT] output path to save the intent/entitymetrics report\r\n", + " --successes [SUCCESSES]\r\n", + " output path to save successful predictions\r\n", + " --errors ERRORS output path to save model errors\r\n", + " --histogram HISTOGRAM\r\n", + " output path for the confidence histogram\r\n", + " --confmat CONFMAT output path for the confusion matrix plot\r\n" ] } ], @@ -242,7 +230,7 @@ "metadata": {}, "source": [ "* O comando abaixo gera informações relevates para a validação das `intents` são elas:\n", - " * Matriz de confuzão\n", + " * Matriz de confusão\n", " * Histograma\n", " * Erros de intents" ] @@ -258,57 +246,55 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\n", " warn(RuntimeWarning(msg))\n", - "2019-05-03 21:20:46.960890: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-05-03 21:20:46.979691: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2905000000 Hz\n", - "2019-05-03 21:20:46.979956: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x565180f44560 executing computations on platform Host. Devices:\n", - "2019-05-03 21:20:46.979989: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-05-03 21:20:47 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:31:33.800517: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:31:33.828980: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:31:33.829499: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55c71b5187e0 executing computations on platform Host. Devices:\n", + "2019-07-17 16:31:33.829554: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:31:34 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use standard file APIs to check for files with this prefix.\u001b[0m\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mtensorflow\u001b[0m - Restoring parameters from ../../bot/models/nlu/current/component_4_EmbeddingIntentClassifier.ckpt\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../bot/data/intents/aleatorio.md is md\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mtensorflow\u001b[0m - Restoring parameters from /models/nlu/current/component_4_EmbeddingIntentClassifier.ckpt\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/geral.md is md\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'cumprimentar', 'diga_mais', 'out_of_scope', 'despedir', 'tudo_bem', 'negar', 'elogios'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/aleatorio.md is md\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 197 (24 distinct intents)\n", - "\t- Found intents: 'de_onde_voce_eh', 'me', 'esporte', 'risada', 'cor', 'signo', 'hobby', 'star_wars', 'linguagens', 'triste', 'historia', 'religiao', 'time', 'filme', 'relationship', 'onde_voce_mora', 'genero', 'playlist', 'bff', 'piada', 'filhos', 'license', 'comida', 'como_estou'\n", + "\t- Found intents: 'relationship', 'me', 'time', 'bff', 'religiao', 'license', 'esporte', 'filhos', 'risada', 'historia', 'de_onde_voce_eh', 'piada', 'genero', 'linguagens', 'cor', 'signo', 'onde_voce_mora', 'playlist', 'filme', 'star_wars', 'triste', 'comida', 'como_estou', 'hobby'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'signo', 'triste', 'linguagens', 'live', 'filme', 'relationship', 'where', 'genero', 'playlist', 'bff', 'starwars', 'piada', 'filhos', 'license', 'comida', 'me', 'esporte', 'cor', 'hobby', 'religiao', 'how', 'historia'\n", + "\t- found entities: 'me', 'bff', 'religiao', 'license', 'piada', 'cor', 'filme', 'triste', 'comida', 'where', 'relationship', 'how', 'live', 'esporte', 'historia', 'filhos', 'genero', 'linguagens', 'signo', 'starwars', 'playlist', 'hobby'\n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../bot/data/intents/actions.md is md\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/actions.md is md\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 6 (1 distinct intents)\n", "\t- Found intents: 'action_test'\n", "\t- entity examples: 0 (0 distinct entities)\n", "\t- found entities: \n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../bot/data/intents/geral.md is md\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 85 (7 distinct intents)\n", - "\t- Found intents: 'elogios', 'cumprimentar', 'out_of_scope', 'despedir', 'negar', 'tudo_bem', 'diga_mais'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 288 (32 distinct intents)\n", - "\t- Found intents: 'action_test', 'elogios', 'out_of_scope', 'signo', 'star_wars', 'linguagens', 'triste', 'diga_mais', 'filme', 'relationship', 'onde_voce_mora', 'genero', 'playlist', 'cumprimentar', 'bff', 'piada', 'filhos', 'license', 'comida', 'como_estou', 'de_onde_voce_eh', 'me', 'esporte', 'risada', 'cor', 'despedir', 'hobby', 'tudo_bem', 'religiao', 'time', 'negar', 'historia'\n", + "\t- Found intents: 'me', 'cumprimentar', 'out_of_scope', 'bff', 'religiao', 'license', 'negar', 'elogios', 'risada', 'de_onde_voce_eh', 'piada', 'cor', 'filme', 'triste', 'comida', 'como_estou', 'relationship', 'action_test', 'despedir', 'hobby', 'esporte', 'filhos', 'historia', 'genero', 'linguagens', 'signo', 'diga_mais', 'playlist', 'tudo_bem', 'onde_voce_mora', 'time', 'star_wars'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'signo', 'triste', 'linguagens', 'live', 'filme', 'relationship', 'where', 'genero', 'playlist', 'bff', 'starwars', 'piada', 'filhos', 'license', 'comida', 'me', 'esporte', 'cor', 'hobby', 'religiao', 'how', 'historia'\n", + "\t- found entities: 'me', 'bff', 'religiao', 'license', 'piada', 'cor', 'filme', 'triste', 'comida', 'where', 'relationship', 'how', 'live', 'esporte', 'historia', 'filhos', 'genero', 'linguagens', 'signo', 'starwars', 'playlist', 'hobby'\n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", - "100%|████████████████████████████████████████| 288/288 [00:00<00:00, 561.88it/s]\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent evaluation results:\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent Evaluation: Only considering those 288 examples that have a defined intent out of 288 examples\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 288/288 [00:00<00:00, 429.64it/s]\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent evaluation results:\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent Evaluation: Only considering those 288 examples that have a defined intent out of 288 examples\n", "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", " 'recall', 'true', average, warn_for)\n", "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", " 'recall', 'true', average, warn_for)\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - F1-Score: 0.9914054566832343\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Precision: 0.9942129629629629\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Accuracy: 0.9895833333333334\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Classification report: \n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - F1-Score: 0.9914054566832343\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Precision: 0.9942129629629629\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Accuracy: 0.9895833333333334\n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Classification report: \n", " precision recall f1-score support\n", "\n", " 0.00 0.00 0.00 0\n", @@ -349,8 +335,10 @@ " macro avg 0.96 0.96 0.96 288\n", " weighted avg 0.99 0.99 0.99 288\n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Model prediction errors saved to errors.json.\n", - "2019-05-03 21:20:49 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Confusion matrix, without normalization: \n", + "2019-07-17 16:31:34 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Model prediction errors saved to errors.json.\n", + "2019-07-17 16:31:35 \u001b[1;30mINFO \u001b[0m \u001b[34mmatplotlib.font_manager\u001b[0m - font search path ['/usr/local/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/ttf', '/usr/local/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/afm', '/usr/local/lib/python3.6/site-packages/matplotlib/mpl-data/fonts/pdfcorefonts']\n", + "2019-07-17 16:31:36 \u001b[1;30mINFO \u001b[0m \u001b[34mmatplotlib.font_manager\u001b[0m - generated new fontManager\n", + "2019-07-17 16:31:38 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Confusion matrix, without normalization: \n", "[[ 0 0 0 ... 0 0 0]\n", " [ 0 6 0 ... 0 0 0]\n", " [ 0 0 11 ... 0 0 0]\n", @@ -360,56 +348,62 @@ " [ 0 0 0 ... 0 0 10]]\n", "Figure(2000x2000)\n", "Figure(1000x1000)\n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity evaluation results:\n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Evaluation for entity extractor: CRFEntityExtractor \n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - F1-Score: 1.0\n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Precision: 1.0\n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Accuracy: 1.0\n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Classification report: \n", - " precision recall f1-score support\n", - "\n", - " bff 1.00 1.00 1.00 17\n", - " comida 1.00 1.00 1.00 17\n", - " cor 1.00 1.00 1.00 13\n", - " esporte 1.00 1.00 1.00 6\n", - " filhos 1.00 1.00 1.00 11\n", - " filme 1.00 1.00 1.00 4\n", - " genero 1.00 1.00 1.00 15\n", - " historia 1.00 1.00 1.00 6\n", - " hobby 1.00 1.00 1.00 11\n", - " how 1.00 1.00 1.00 24\n", - " license 1.00 1.00 1.00 14\n", - " linguagens 1.00 1.00 1.00 14\n", - " live 1.00 1.00 1.00 13\n", - " me 1.00 1.00 1.00 6\n", - " no_entity 1.00 1.00 1.00 613\n", - " piada 1.00 1.00 1.00 7\n", - " playlist 1.00 1.00 1.00 7\n", - "relationship 1.00 1.00 1.00 9\n", - " religiao 1.00 1.00 1.00 16\n", - " signo 1.00 1.00 1.00 3\n", - " starwars 1.00 1.00 1.00 13\n", - " triste 1.00 1.00 1.00 7\n", - " where 1.00 1.00 1.00 3\n", - "\n", - " micro avg 1.00 1.00 1.00 849\n", - " macro avg 1.00 1.00 1.00 849\n", - "weighted avg 1.00 1.00 1.00 849\n", - "\n", - "2019-05-03 21:20:54 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished evaluation\n" + "2019-07-17 16:31:43 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity evaluation results:\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Evaluation for entity extractor: CRFEntityExtractor \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - F1-Score: 1.0\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Precision: 1.0\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Accuracy: 1.0\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Classification report: \r\n", + " precision recall f1-score support\r\n", + "\r\n", + " bff 1.00 1.00 1.00 17\r\n", + " comida 1.00 1.00 1.00 17\r\n", + " cor 1.00 1.00 1.00 13\r\n", + " esporte 1.00 1.00 1.00 6\r\n", + " filhos 1.00 1.00 1.00 11\r\n", + " filme 1.00 1.00 1.00 4\r\n", + " genero 1.00 1.00 1.00 15\r\n", + " historia 1.00 1.00 1.00 6\r\n", + " hobby 1.00 1.00 1.00 11\r\n", + " how 1.00 1.00 1.00 24\r\n", + " license 1.00 1.00 1.00 14\r\n", + " linguagens 1.00 1.00 1.00 14\r\n", + " live 1.00 1.00 1.00 13\r\n", + " me 1.00 1.00 1.00 6\r\n", + " no_entity 1.00 1.00 1.00 613\r\n", + " piada 1.00 1.00 1.00 7\r\n", + " playlist 1.00 1.00 1.00 7\r\n", + "relationship 1.00 1.00 1.00 9\r\n", + " religiao 1.00 1.00 1.00 16\r\n", + " signo 1.00 1.00 1.00 3\r\n", + " starwars 1.00 1.00 1.00 13\r\n", + " triste 1.00 1.00 1.00 7\r\n", + " where 1.00 1.00 1.00 3\r\n", + "\r\n", + " micro avg 1.00 1.00 1.00 849\r\n", + " macro avg 1.00 1.00 1.00 849\r\n", + "weighted avg 1.00 1.00 1.00 849\r\n", + "\r\n", + "2019-07-17 16:31:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished evaluation\r\n" ] } ], "source": [ - "!python -m rasa_nlu.test -d $BOT_INTENTS_PATH -m $BOT_MODELS_NLU_PATH --mode evaluation" + "!python -m rasa_nlu.test -d $COACH_INTENTS_PATH -m $COACH_MODELS_NLU_PATH --mode evaluation" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Matriz de Confuzão\n", - "* A matriz de confuzão mostra a correlação entre as intents.\n", + "#### Matriz de Confusão\n", + "* A matriz de confusão mostra a correlação entre as intents.\n", "* A diagonal principal tem forte correlação pois mostra a relação de uma intent **com ela mesma**\n", "* O ideal é que não haja **nenhum valor** diferente de **0 fora da diagonal principal**." ] @@ -435,7 +429,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 5, @@ -452,7 +446,7 @@ "metadata": {}, "source": [ "#### Erros\n", - "* O arquivo `erros,json` mostra os erros encontrados. Este arquivo lista os mesmos erros mostrados na **matriz de confuzão**, então caso ela fique muito grande, não se preocupe, pois você pode apenas procurar os erros no arquivo gerado.\n", + "* O arquivo `erros,json` mostra os erros encontrados. Este arquivo lista os mesmos erros mostrados na **matriz de confusão**, então caso ela fique muito grande, não se preocupe, pois você pode apenas procurar os erros no arquivo gerado.\n", "* Os erros mostrados são textos repetidos nos exemplos de diferrentes `intents`.\n", "* Caso o arquivo não seja gerado significa que não foram encontrados erros." ] @@ -470,19 +464,11 @@ "text": [ "[\r\n", " {\r\n", - " \"text\": \"e o c#\",\r\n", - " \"intent\": \"linguagens\",\r\n", - " \"intent_prediction\": {\r\n", - " \"name\": \"\",\r\n", - " \"confidence\": 0.0\r\n", - " }\r\n", - " },\r\n", - " {\r\n", " \"text\": \"como vai\",\r\n", " \"intent\": \"cumprimentar\",\r\n", " \"intent_prediction\": {\r\n", " \"name\": \"tudo_bem\",\r\n", - " \"confidence\": 0.835846483707428\r\n", + " \"confidence\": 0.7145657539367676\r\n", " }\r\n", " },\r\n", " {\r\n", @@ -490,7 +476,15 @@ " \"intent\": \"cumprimentar\",\r\n", " \"intent_prediction\": {\r\n", " \"name\": \"tudo_bem\",\r\n", - " \"confidence\": 0.8136337995529175\r\n", + " \"confidence\": 0.7155274152755737\r\n", + " }\r\n", + " },\r\n", + " {\r\n", + " \"text\": \"e o c#\",\r\n", + " \"intent\": \"linguagens\",\r\n", + " \"intent_prediction\": {\r\n", + " \"name\": \"\",\r\n", + " \"confidence\": 0.0\r\n", " }\r\n", " }\r\n", "]" @@ -531,7 +525,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 7, @@ -561,64 +555,62 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.test' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.test'; this may result in unpredictable behaviour\n", " warn(RuntimeWarning(msg))\n", - "2019-05-03 21:20:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../bot/data/intents/aleatorio.md is md\n", - "2019-05-03 21:20:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/geral.md is md\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'cumprimentar', 'tudo_bem', 'negar', 'elogios', 'despedir', 'out_of_scope', 'diga_mais'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/aleatorio.md is md\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 197 (24 distinct intents)\n", - "\t- Found intents: 'linguagens', 'triste', 'cor', 'hobby', 'risada', 'bff', 'genero', 'star_wars', 'religiao', 'filhos', 'como_estou', 'me', 'signo', 'historia', 'relationship', 'filme', 'playlist', 'time', 'license', 'piada', 'comida', 'onde_voce_mora', 'esporte', 'de_onde_voce_eh'\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'comida', 'cor', 'historia', 'triste', 'como_estou', 'filme', 'star_wars', 'relationship', 'esporte', 'signo', 'me', 'linguagens', 'de_onde_voce_eh', 'religiao', 'hobby', 'playlist', 'license', 'onde_voce_mora'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'linguagens', 'how', 'hobby', 'starwars', 'filhos', 'me', 'signo', 'relationship', 'where', 'playlist', 'filme', 'license', 'live', 'triste', 'cor', 'bff', 'genero', 'religiao', 'historia', 'piada', 'comida', 'esporte'\n", + "\t- found entities: 'live', 'filhos', 'genero', 'relationship', 'linguagens', 'me', 'hobby', 'starwars', 'playlist', 'license', 'bff', 'piada', 'comida', 'cor', 'historia', 'triste', 'where', 'filme', 'how', 'esporte', 'signo', 'religiao'\n", "\n", - "2019-05-03 21:20:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../bot/data/intents/actions.md is md\n", - "2019-05-03 21:20:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../coach/data/intents/actions.md is md\n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 6 (1 distinct intents)\n", "\t- Found intents: 'action_test'\n", "\t- entity examples: 0 (0 distinct entities)\n", "\t- found entities: \n", "\n", - "2019-05-03 21:20:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of ../../bot/data/intents/geral.md is md\n", - "2019-05-03 21:20:56 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 85 (7 distinct intents)\n", - "\t- Found intents: 'despedir', 'diga_mais', 'cumprimentar', 'negar', 'elogios', 'tudo_bem', 'out_of_scope'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "2019-05-03 21:20:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 288 (32 distinct intents)\n", - "\t- Found intents: 'linguagens', 'hobby', 'star_wars', 'filhos', 'out_of_scope', 'me', 'signo', 'relationship', 'filme', 'playlist', 'license', 'elogios', 'tudo_bem', 'action_test', 'de_onde_voce_eh', 'despedir', 'triste', 'cor', 'risada', 'negar', 'bff', 'genero', 'religiao', 'como_estou', 'historia', 'time', 'diga_mais', 'piada', 'cumprimentar', 'comida', 'onde_voce_mora', 'esporte'\n", + "\t- Found intents: 'filhos', 'genero', 'time', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'tudo_bem', 'linguagens', 'me', 'hobby', 'playlist', 'license', 'onde_voce_mora', 'bff', 'risada', 'piada', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'elogios', 'filme', 'action_test', 'esporte', 'signo', 'de_onde_voce_eh', 'despedir', 'religiao', 'diga_mais'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'linguagens', 'how', 'hobby', 'starwars', 'filhos', 'me', 'signo', 'relationship', 'where', 'playlist', 'filme', 'license', 'live', 'triste', 'cor', 'bff', 'genero', 'religiao', 'historia', 'piada', 'comida', 'esporte'\n", + "\t- found entities: 'live', 'filhos', 'genero', 'relationship', 'linguagens', 'me', 'hobby', 'starwars', 'playlist', 'license', 'bff', 'piada', 'comida', 'cor', 'historia', 'triste', 'where', 'filme', 'how', 'esporte', 'signo', 'religiao'\n", "\n", - "2019-05-03 21:20:57 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:19 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 273 (28 distinct intents)\n", - "\t- Found intents: 'linguagens', 'despedir', 'triste', 'cor', 'hobby', 'risada', 'negar', 'bff', 'genero', 'star_wars', 'religiao', 'filhos', 'out_of_scope', 'como_estou', 'me', 'historia', 'relationship', 'playlist', 'time', 'license', 'diga_mais', 'piada', 'cumprimentar', 'comida', 'tudo_bem', 'onde_voce_mora', 'esporte', 'action_test'\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", "\t- entity examples: 162 (19 distinct entities)\n", - "\t- found entities: 'triste', 'linguagens', 'live', 'license', 'me', 'cor', 'hobby', 'how', 'bff', 'comida', 'genero', 'piada', 'religiao', 'filhos', 'starwars', 'historia', 'esporte', 'relationship', 'playlist'\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", "\n", - "2019-05-03 21:20:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 131 (28 distinct intents)\n", - "\t- Found intents: 'linguagens', 'despedir', 'triste', 'cor', 'hobby', 'risada', 'negar', 'bff', 'genero', 'star_wars', 'religiao', 'filhos', 'out_of_scope', 'como_estou', 'me', 'historia', 'relationship', 'playlist', 'time', 'license', 'diga_mais', 'piada', 'cumprimentar', 'comida', 'tudo_bem', 'onde_voce_mora', 'esporte', 'action_test'\n", - "\t- entity examples: 76 (19 distinct entities)\n", - "\t- found entities: 'triste', 'linguagens', 'live', 'license', 'me', 'cor', 'hobby', 'how', 'bff', 'comida', 'genero', 'piada', 'religiao', 'filhos', 'starwars', 'historia', 'esporte', 'relationship', 'playlist'\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 79 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", "\n", - "2019-05-03 21:20:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 142 (28 distinct intents)\n", - "\t- Found intents: 'linguagens', 'despedir', 'triste', 'cor', 'hobby', 'risada', 'negar', 'bff', 'genero', 'star_wars', 'religiao', 'filhos', 'out_of_scope', 'como_estou', 'me', 'historia', 'relationship', 'playlist', 'time', 'license', 'diga_mais', 'piada', 'cumprimentar', 'comida', 'tudo_bem', 'onde_voce_mora', 'esporte', 'action_test'\n", - "\t- entity examples: 86 (19 distinct entities)\n", - "\t- found entities: 'triste', 'linguagens', 'live', 'license', 'me', 'cor', 'hobby', 'how', 'bff', 'comida', 'genero', 'piada', 'religiao', 'filhos', 'starwars', 'historia', 'esporte', 'relationship', 'playlist'\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 83 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", "\n", - "2019-05-03 21:20:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", - "2019-05-03 21:20:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:58 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", - "2019-05-03 21:20:59 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:59 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", - "2019-05-03 21:20:59 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:59 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", - "2019-05-03 21:20:59 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:59 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:20 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", "For more information, please see:\n", @@ -626,22 +618,22 @@ " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n", - "2019-05-03 21:20:59 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use keras.layers.dense instead.\u001b[0m\n", - "2019-05-03 21:20:59 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\u001b[0m\n", - "2019-05-03 21:20:59 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use keras.layers.dropout instead.\u001b[0m\n", - "2019-05-03 21:20:59 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "2019-07-17 16:32:21 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", - "2019-05-03 21:21:00 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:32:22 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use tf.cast instead.\u001b[0m\n", - "2019-05-03 21:21:00 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:32:22 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n" ] @@ -650,80 +642,90 @@ "name": "stdout", "output_type": "stream", "text": [ - "2019-05-03 21:21:00.706185: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-05-03 21:21:00.726392: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2905000000 Hz\n", - "2019-05-03 21:21:00.726674: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x555b9ee46db0 executing computations on platform Host. Devices:\n", - "2019-05-03 21:21:00.726702: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-05-03 21:21:00 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:05<00:00, 59.85it/s, loss=0.186, acc=1.000]\n", - "2019-05-03 21:21:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.186, train accuracy=1.000\n", - "2019-05-03 21:21:05 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:05 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", - "100%|████████████████████████████████████████| 131/131 [00:00<00:00, 353.93it/s]\n", - "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", - " 'recall', 'true', average, warn_for)\n", - "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", - " 'recall', 'true', average, warn_for)\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", - "100%|████████████████████████████████████████| 142/142 [00:00<00:00, 528.87it/s]\n", + "2019-07-17 16:32:22.678227: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:32:22.700933: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:32:22.701522: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55be5a7c66c0 executing computations on platform Host. Devices:\n", + "2019-07-17 16:32:22.701579: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:32:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:05<00:00, 53.69it/s, loss=0.194, acc=0.992]\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.194, train accuracy=0.992\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 131/131 [00:00<00:00, 435.20it/s]\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 142/142 [00:00<00:00, 527.33it/s]\n", "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n", " 'precision', 'predicted', average, warn_for)\n", "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n", " 'precision', 'predicted', average, warn_for)\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 142 (28 distinct intents)\n", - "\t- Found intents: 'linguagens', 'despedir', 'triste', 'cor', 'hobby', 'risada', 'negar', 'bff', 'genero', 'star_wars', 'religiao', 'filhos', 'out_of_scope', 'como_estou', 'me', 'historia', 'relationship', 'playlist', 'time', 'license', 'diga_mais', 'piada', 'cumprimentar', 'comida', 'tudo_bem', 'onde_voce_mora', 'esporte', 'action_test'\n", - "\t- entity examples: 86 (19 distinct entities)\n", - "\t- found entities: 'triste', 'linguagens', 'live', 'license', 'me', 'cor', 'hobby', 'how', 'bff', 'comida', 'genero', 'piada', 'religiao', 'filhos', 'starwars', 'historia', 'esporte', 'relationship', 'playlist'\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 83 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", "\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:32:28 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 131 (28 distinct intents)\n", - "\t- Found intents: 'linguagens', 'despedir', 'triste', 'cor', 'hobby', 'risada', 'negar', 'bff', 'genero', 'star_wars', 'religiao', 'filhos', 'out_of_scope', 'como_estou', 'me', 'historia', 'relationship', 'playlist', 'time', 'license', 'diga_mais', 'piada', 'cumprimentar', 'comida', 'tudo_bem', 'onde_voce_mora', 'esporte', 'action_test'\n", - "\t- entity examples: 76 (19 distinct entities)\n", - "\t- found entities: 'triste', 'linguagens', 'live', 'license', 'me', 'cor', 'hobby', 'how', 'bff', 'comida', 'genero', 'piada', 'religiao', 'filhos', 'starwars', 'historia', 'esporte', 'relationship', 'playlist'\n", + "\t- Found intents: 'bff', 'risada', 'filhos', 'genero', 'piada', 'time', 'negar', 'comida', 'cor', 'historia', 'out_of_scope', 'triste', 'como_estou', 'cumprimentar', 'star_wars', 'relationship', 'hobby', 'action_test', 'tudo_bem', 'esporte', 'linguagens', 'despedir', 'religiao', 'me', 'playlist', 'license', 'onde_voce_mora', 'diga_mais'\n", + "\t- entity examples: 79 (19 distinct entities)\n", + "\t- found entities: 'bff', 'triste', 'live', 'filhos', 'genero', 'how', 'piada', 'relationship', 'esporte', 'comida', 'me', 'cor', 'historia', 'linguagens', 'religiao', 'hobby', 'starwars', 'playlist', 'license'\n", "\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", - "2019-05-03 21:21:07 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:06<00:00, 47.98it/s, loss=0.186, acc=1.000]\n", - "2019-05-03 21:21:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.186, train accuracy=1.000\n", - "2019-05-03 21:21:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:13 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", - "100%|████████████████████████████████████████| 142/142 [00:00<00:00, 520.96it/s]\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", - "100%|████████████████████████████████████████| 131/131 [00:00<00:00, 595.66it/s]\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - CV evaluation (n=2)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent evaluation results\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Accuracy: 0.996 (0.004)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train F1-score: 0.998 (0.002)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Precision: 1.000 (0.000)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Accuracy: 0.523 (0.012)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test F1-score: 0.545 (0.007)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Precision: 0.641 (0.014)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity evaluation results\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity extractor: CRFEntityExtractor\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Accuracy: 1.000 (0.000)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train F1-score: 1.000 (0.000)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Precision: 1.000 (0.000)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity extractor: CRFEntityExtractor\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Accuracy: 0.834 (0.015)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test F1-score: 0.788 (0.013)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Precision: 0.800 (0.015)\n", - "2019-05-03 21:21:14 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished evaluation\n" + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:29 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "2019-07-17 16:32:30 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:06<00:00, 45.43it/s, loss=0.188, acc=1.000]\n", + "2019-07-17 16:32:36 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.188, train accuracy=1.000\n", + "2019-07-17 16:32:36 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:32:36 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 142/142 [00:00<00:00, 433.04it/s]\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1145: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no true samples.\n", + " 'recall', 'true', average, warn_for)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Running model for predictions:\n", + "100%|████████████████████████████████████████| 131/131 [00:00<00:00, 569.47it/s]\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "/usr/local/lib/python3.6/site-packages/sklearn/metrics/classification.py:1143: UndefinedMetricWarning: F-score is ill-defined and being set to 0.0 in labels with no predicted samples.\n", + " 'precision', 'predicted', average, warn_for)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - CV evaluation (n=2)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Intent evaluation results\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Accuracy: 0.993 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train F1-score: 0.994 (0.002)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Precision: 0.997 (0.003)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Accuracy: 0.536 (0.029)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test F1-score: 0.561 (0.031)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Precision: 0.681 (0.027)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity evaluation results\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity extractor: CRFEntityExtractor\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Accuracy: 1.000 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train F1-score: 1.000 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - train Precision: 1.000 (0.000)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Entity extractor: CRFEntityExtractor\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Accuracy: 0.826 (0.004)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test F1-score: 0.776 (0.001)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - test Precision: 0.810 (0.033)\n", + "2019-07-17 16:32:37 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished evaluation\n" ] } ], "source": [ - "!python -m rasa_nlu.test -d $BOT_INTENTS_PATH -c $BOT_NLU_CONFIG_PATH --mode crossvalidation --folds 2 --report" + "!python -m rasa_nlu.test -d $COACH_INTENTS_PATH -c $COACH_NLU_CONFIG_PATH --mode crossvalidation --folds 2 --report" ] }, { @@ -782,7 +784,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.9" } }, "nbformat": 4, diff --git a/notebooks/stories/chat_graph.html b/notebooks/stories/chat_graph.html index 6b9897e9..ebffb162 100644 --- a/notebooks/stories/chat_graph.html +++ b/notebooks/stories/chat_graph.html @@ -137,72 +137,72 @@ graph = `digraph { 0 [class="start active", fillcolor=green, fontsize=12, label=START, style=filled]; "-1" [class=end, fillcolor=red, fontsize=12, label=END, style=filled]; -1 [class="", fontsize=12, label=utter_como_estou]; -2 [class="", fontsize=12, label=utter_playlist]; -3 [class="", fontsize=12, label=utter_comida]; -4 [class="", fontsize=12, label=utter_cor]; -5 [class="", fontsize=12, label=utter_de_onde_voce_eh]; -6 [class="", fontsize=12, label=utter_relationship]; -7 [class="", fontsize=12, label=utter_me]; -8 [class="", fontsize=12, label=utter_filhos]; -9 [class="", fontsize=12, label=utter_filme]; -10 [class="", fontsize=12, label=utter_signo]; -11 [class="", fontsize=12, label=utter_triste]; -12 [class="", fontsize=12, label=utter_religiao]; -13 [class="", fontsize=12, label=utter_hobby]; -14 [class="", fontsize=12, label=utter_bff]; -15 [class="", fontsize=12, label=utter_historia]; -16 [class="", fontsize=12, label=utter_risada]; -17 [class="", fontsize=12, label=utter_cumprimentar]; -18 [class="", fontsize=12, label=action_test]; -20 [class="", fontsize=12, label=utter_despedir]; -21 [class="", fontsize=12, label=utter_tudo_bem]; -24 [class="", fontsize=12, label=utter_diga_mais]; -25 [class="", fontsize=12, label=utter_esporte]; -26 [class="", fontsize=12, label=utter_default]; -28 [class="", fontsize=12, label=utter_elogios]; -32 [class="", fontsize=12, label=utter_time]; -33 [class="", fontsize=12, label=utter_linguagens]; -34 [class="", fontsize=12, label=utter_genero]; -35 [class="", fontsize=12, label=utter_star_wars]; -36 [class="", fontsize=12, label=utter_piada]; -37 [class="", fontsize=12, label=utter_license]; -38 [class="", fontsize=12, label=utter_onde_voce_mora]; -39 [class=intent, fillcolor=lightblue, label=como_estou, shape=rect, style=filled]; -40 [class=intent, fillcolor=lightblue, label=playlist, shape=rect, style=filled]; -41 [class=intent, fillcolor=lightblue, label=comida, shape=rect, style=filled]; -42 [class=intent, fillcolor=lightblue, label=cor, shape=rect, style=filled]; -43 [class=intent, fillcolor=lightblue, label=de_onde_voce_eh, shape=rect, style=filled]; -44 [class=intent, fillcolor=lightblue, label=me, shape=rect, style=filled]; -45 [class=intent, fillcolor=lightblue, label=filhos, shape=rect, style=filled]; -46 [class=intent, fillcolor=lightblue, label=filme, shape=rect, style=filled]; -47 [class=intent, fillcolor=lightblue, label=signo, shape=rect, style=filled]; -48 [class=intent, fillcolor=lightblue, label=triste, shape=rect, style=filled]; -49 [class=intent, fillcolor=lightblue, label=religiao, shape=rect, style=filled]; -50 [class=intent, fillcolor=lightblue, label=hobby, shape=rect, style=filled]; -51 [class=intent, fillcolor=lightblue, label=bff, shape=rect, style=filled]; -52 [class=intent, fillcolor=lightblue, label=historia, shape=rect, style=filled]; -53 [class=intent, fillcolor=lightblue, label=risada, shape=rect, style=filled]; -54 [class=intent, fillcolor=lightblue, label=cumprimentar, shape=rect, style=filled]; -55 [class=intent, fillcolor=lightblue, label=despedir, shape=rect, style=filled]; -56 [class=intent, fillcolor=lightblue, label=negar, shape=rect, style=filled]; -57 [class=intent, fillcolor=lightblue, label=tudo_bem, shape=rect, style=filled]; -58 [class=intent, fillcolor=lightblue, label=diga_mais, shape=rect, style=filled]; -59 [class=intent, fillcolor=lightblue, label=esporte, shape=rect, style=filled]; -60 [class=intent, fillcolor=lightblue, label=out_of_scope, shape=rect, style=filled]; -61 [class=intent, fillcolor=lightblue, label=elogios, shape=rect, style=filled]; -62 [class=intent, fillcolor=lightblue, label=time, shape=rect, style=filled]; -63 [class=intent, fillcolor=lightblue, label=linguagens, shape=rect, style=filled]; -64 [class=intent, fillcolor=lightblue, label=genero, shape=rect, style=filled]; -65 [class=intent, fillcolor=lightblue, label=star_wars, shape=rect, style=filled]; -66 [class=intent, fillcolor=lightblue, label=piada, shape=rect, style=filled]; -67 [class=intent, fillcolor=lightblue, label=license, shape=rect, style=filled]; -68 [class=intent, fillcolor=lightblue, label=onde_voce_mora, shape=rect, style=filled]; +1 [class="", fontsize=12, label=utter_religiao]; +2 [class="", fontsize=12, label=utter_esporte]; +3 [class="", fontsize=12, label=utter_time]; +4 [class="", fontsize=12, label=utter_linguagens]; +5 [class="", fontsize=12, label=utter_genero]; +6 [class="", fontsize=12, label=utter_star_wars]; +7 [class="", fontsize=12, label=utter_piada]; +8 [class="", fontsize=12, label=utter_license]; +9 [class="", fontsize=12, label=utter_onde_voce_mora]; +10 [class="", fontsize=12, label=utter_como_estou]; +11 [class="", fontsize=12, label=utter_despedir]; +12 [class="", fontsize=12, label=utter_playlist]; +13 [class="", fontsize=12, label=utter_comida]; +14 [class="", fontsize=12, label=utter_cor]; +15 [class="", fontsize=12, label=utter_de_onde_voce_eh]; +16 [class="", fontsize=12, label=utter_relationship]; +17 [class="", fontsize=12, label=utter_me]; +18 [class="", fontsize=12, label=utter_filhos]; +19 [class="", fontsize=12, label=utter_filme]; +20 [class="", fontsize=12, label=utter_signo]; +21 [class="", fontsize=12, label=utter_triste]; +22 [class="", fontsize=12, label=utter_hobby]; +23 [class="", fontsize=12, label=utter_tudo_bem]; +24 [class="", fontsize=12, label=utter_bff]; +25 [class="", fontsize=12, label=utter_historia]; +26 [class="", fontsize=12, label=utter_risada]; +27 [class="", fontsize=12, label=utter_cumprimentar]; +28 [class="", fontsize=12, label=action_test]; +32 [class="", fontsize=12, label=utter_diga_mais]; +33 [class="", fontsize=12, label=utter_default]; +35 [class="", fontsize=12, label=utter_elogios]; +39 [class=intent, fillcolor=lightblue, label=religiao, shape=rect, style=filled]; +40 [class=intent, fillcolor=lightblue, label=esporte, shape=rect, style=filled]; +41 [class=intent, fillcolor=lightblue, label=time, shape=rect, style=filled]; +42 [class=intent, fillcolor=lightblue, label=linguagens, shape=rect, style=filled]; +43 [class=intent, fillcolor=lightblue, label=genero, shape=rect, style=filled]; +44 [class=intent, fillcolor=lightblue, label=star_wars, shape=rect, style=filled]; +45 [class=intent, fillcolor=lightblue, label=piada, shape=rect, style=filled]; +46 [class=intent, fillcolor=lightblue, label=license, shape=rect, style=filled]; +47 [class=intent, fillcolor=lightblue, label=onde_voce_mora, shape=rect, style=filled]; +48 [class=intent, fillcolor=lightblue, label=como_estou, shape=rect, style=filled]; +49 [class=intent, fillcolor=lightblue, label=despedir, shape=rect, style=filled]; +50 [class=intent, fillcolor=lightblue, label=negar, shape=rect, style=filled]; +51 [class=intent, fillcolor=lightblue, label=playlist, shape=rect, style=filled]; +52 [class=intent, fillcolor=lightblue, label=comida, shape=rect, style=filled]; +53 [class=intent, fillcolor=lightblue, label=cor, shape=rect, style=filled]; +54 [class=intent, fillcolor=lightblue, label=de_onde_voce_eh, shape=rect, style=filled]; +55 [class=intent, fillcolor=lightblue, label=me, shape=rect, style=filled]; +56 [class=intent, fillcolor=lightblue, label=filhos, shape=rect, style=filled]; +57 [class=intent, fillcolor=lightblue, label=filme, shape=rect, style=filled]; +58 [class=intent, fillcolor=lightblue, label=signo, shape=rect, style=filled]; +59 [class=intent, fillcolor=lightblue, label=triste, shape=rect, style=filled]; +60 [class=intent, fillcolor=lightblue, label=hobby, shape=rect, style=filled]; +61 [class=intent, fillcolor=lightblue, label=tudo_bem, shape=rect, style=filled]; +62 [class=intent, fillcolor=lightblue, label=bff, shape=rect, style=filled]; +63 [class=intent, fillcolor=lightblue, label=historia, shape=rect, style=filled]; +64 [class=intent, fillcolor=lightblue, label=risada, shape=rect, style=filled]; +65 [class=intent, fillcolor=lightblue, label=cumprimentar, shape=rect, style=filled]; +66 [class=intent, fillcolor=lightblue, label=diga_mais, shape=rect, style=filled]; +67 [class=intent, fillcolor=lightblue, label=out_of_scope, shape=rect, style=filled]; +68 [class=intent, fillcolor=lightblue, label=elogios, shape=rect, style=filled]; 69 [class=intent, fillcolor=lightblue, label=action_test, shape=rect, style=filled]; 70 [class=intent, fillcolor=lightblue, label=relationship, shape=rect, style=filled]; 71 [class=intent, fillcolor=lightblue, label=action_test, shape=rect, style=filled]; -72 [class=intent, fillcolor=lightblue, label=despedir, shape=rect, style=filled]; -73 [class=intent, fillcolor=lightblue, label=tudo_bem, shape=rect, style=filled]; +72 [class=intent, fillcolor=lightblue, label=tudo_bem, shape=rect, style=filled]; +73 [class=intent, fillcolor=lightblue, label=despedir, shape=rect, style=filled]; 0 -> 39 [class="", key=0]; 0 -> 40 [class="", key=0]; 0 -> 41 [class="", key=0]; @@ -238,7 +238,7 @@ 2 -> "-1" [class="", key=NONE, label=""]; 3 -> "-1" [class="", key=NONE, label=""]; 4 -> "-1" [class="", key=NONE, label=""]; -5 -> 70 [class="", key=0]; +5 -> "-1" [class="", key=NONE, label=""]; 6 -> "-1" [class="", key=NONE, label=""]; 7 -> "-1" [class="", key=NONE, label=""]; 8 -> "-1" [class="", key=NONE, label=""]; @@ -248,61 +248,61 @@ 12 -> "-1" [class="", key=NONE, label=""]; 13 -> "-1" [class="", key=NONE, label=""]; 14 -> "-1" [class="", key=NONE, label=""]; -15 -> "-1" [class="", key=NONE, label=""]; +15 -> 70 [class="", key=0]; 16 -> "-1" [class="", key=NONE, label=""]; 17 -> "-1" [class="", key=NONE, label=""]; -17 -> 71 [class="", key=0]; -17 -> 72 [class="", key=0]; -17 -> 73 [class="", key=0]; 18 -> "-1" [class="", key=NONE, label=""]; +19 -> "-1" [class="", key=NONE, label=""]; 20 -> "-1" [class="", key=NONE, label=""]; 21 -> "-1" [class="", key=NONE, label=""]; +22 -> "-1" [class="", key=NONE, label=""]; +23 -> "-1" [class="", key=NONE, label=""]; 24 -> "-1" [class="", key=NONE, label=""]; 25 -> "-1" [class="", key=NONE, label=""]; 26 -> "-1" [class="", key=NONE, label=""]; +27 -> "-1" [class="", key=NONE, label=""]; +27 -> 71 [class="", key=0]; +27 -> 72 [class="", key=0]; +27 -> 73 [class="", key=0]; 28 -> "-1" [class="", key=NONE, label=""]; 32 -> "-1" [class="", key=NONE, label=""]; 33 -> "-1" [class="", key=NONE, label=""]; -34 -> "-1" [class="", key=NONE, label=""]; 35 -> "-1" [class="", key=NONE, label=""]; -36 -> "-1" [class="", key=NONE, label=""]; -37 -> "-1" [class="", key=NONE, label=""]; -38 -> "-1" [class="", key=NONE, label=""]; 39 -> 1 [class="", key=0]; 40 -> 2 [class="", key=0]; 41 -> 3 [class="", key=0]; 42 -> 4 [class="", key=0]; 43 -> 5 [class="", key=0]; -44 -> 7 [class="", key=0]; -45 -> 8 [class="", key=0]; -46 -> 9 [class="", key=0]; -47 -> 10 [class="", key=0]; -48 -> 11 [class="", key=0]; -49 -> 12 [class="", key=0]; -50 -> 13 [class="", key=0]; -51 -> 14 [class="", key=0]; -52 -> 15 [class="", key=0]; -53 -> 16 [class="", key=0]; -54 -> 17 [class="", key=0]; -55 -> 20 [class="", key=0]; -56 -> 20 [class="", key=0]; -57 -> 21 [class="", key=0]; -58 -> 24 [class="", key=0]; -59 -> 25 [class="", key=0]; -60 -> 26 [class="", key=0]; -61 -> 28 [class="", key=0]; -62 -> 32 [class="", key=0]; -63 -> 33 [class="", key=0]; -64 -> 34 [class="", key=0]; -65 -> 35 [class="", key=0]; -66 -> 36 [class="", key=0]; -67 -> 37 [class="", key=0]; -68 -> 38 [class="", key=0]; -69 -> 18 [class="", key=0]; -70 -> 6 [class="", key=0]; -71 -> 18 [class="", key=0]; -72 -> 20 [class="", key=0]; -73 -> 21 [class="", key=0]; +44 -> 6 [class="", key=0]; +45 -> 7 [class="", key=0]; +46 -> 8 [class="", key=0]; +47 -> 9 [class="", key=0]; +48 -> 10 [class="", key=0]; +49 -> 11 [class="", key=0]; +50 -> 11 [class="", key=0]; +51 -> 12 [class="", key=0]; +52 -> 13 [class="", key=0]; +53 -> 14 [class="", key=0]; +54 -> 15 [class="", key=0]; +55 -> 17 [class="", key=0]; +56 -> 18 [class="", key=0]; +57 -> 19 [class="", key=0]; +58 -> 20 [class="", key=0]; +59 -> 21 [class="", key=0]; +60 -> 22 [class="", key=0]; +61 -> 23 [class="", key=0]; +62 -> 24 [class="", key=0]; +63 -> 25 [class="", key=0]; +64 -> 26 [class="", key=0]; +65 -> 27 [class="", key=0]; +66 -> 32 [class="", key=0]; +67 -> 33 [class="", key=0]; +68 -> 35 [class="", key=0]; +69 -> 28 [class="", key=0]; +70 -> 16 [class="", key=0]; +71 -> 28 [class="", key=0]; +72 -> 23 [class="", key=0]; +73 -> 11 [class="", key=0]; } `; drawGraph(graph); diff --git a/notebooks/stories/stories-analysis.ipynb b/notebooks/stories/stories-analysis.ipynb index 7a6b2388..c2ad4edf 100644 --- a/notebooks/stories/stories-analysis.ipynb +++ b/notebooks/stories/stories-analysis.ipynb @@ -24,14 +24,6 @@ "execution_count": 1, "metadata": {}, "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n" - ] - }, { "name": "stdout", "output_type": "stream", @@ -83,11 +75,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_core.visualize' found in sys.modules after import of package 'rasa_core', but prior to execution of 'rasa_core.visualize'; this may result in unpredictable behaviour\n", " warn(RuntimeWarning(msg))\n", - "2019-05-03 21:20:43 \u001b[1;30mINFO \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Scheduler started\n", + "2019-07-17 16:33:49 \u001b[1;30mINFO \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Scheduler started\n", "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", "For more information, please see:\n", @@ -95,14 +85,14 @@ " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n", - "2019-05-03 21:20:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Starting to visualize stories...\n", - "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 5308.85it/s, # trackers=1]\n", - "2019-05-03 21:20:44 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished graph creation. Saved into file:///work/notebooks/stories/chat_graph.html\n" + "2019-07-17 16:33:50 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Starting to visualize stories...\n", + "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 3467.21it/s, # trackers=1]\n", + "2019-07-17 16:33:51 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished graph creation. Saved into file:///work/notebooks/stories/chat_graph.html\n" ] } ], "source": [ - "!python -m rasa_core.visualize -d $BOT_DOMAIN_PATH -s $BOT_STORIES_PATH -o chat_graph.html" + "!python -m rasa_core.visualize -d $COACH_DOMAIN_PATH -s $COACH_STORIES_PATH -o chat_graph.html" ] }, { @@ -135,7 +125,7 @@ " " ], "text/plain": [ - "" + "" ] }, "execution_count": 3, @@ -172,49 +162,47 @@ "name": "stdout", "output_type": "stream", "text": [ - "make: Entering directory '/work/bot'\n", + "make: Entering directory '/work/coach'\n", "python3 -m rasa_nlu.train -c nlu_config.yml --fixed_model_name current \\\n", - " --data data/intents/ -o models --project nlu --verbose\n", - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", + "--data data/intents/ -o /src_models --project nlu --verbose\n", "/usr/local/lib/python3.6/runpy.py:125: RuntimeWarning: 'rasa_nlu.train' found in sys.modules after import of package 'rasa_nlu', but prior to execution of 'rasa_nlu.train'; this may result in unpredictable behaviour\n", " warn(RuntimeWarning(msg))\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/aleatorio.md is md\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/geral.md is md\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "\t- intent examples: 85 (7 distinct intents)\n", + "\t- Found intents: 'diga_mais', 'elogios', 'cumprimentar', 'out_of_scope', 'despedir', 'tudo_bem', 'negar'\n", + "\t- entity examples: 0 (0 distinct entities)\n", + "\t- found entities: \n", + "\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/aleatorio.md is md\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 197 (24 distinct intents)\n", - "\t- Found intents: 'religiao', 'linguagens', 'license', 'como_estou', 'filme', 'hobby', 'de_onde_voce_eh', 'signo', 'filhos', 'relationship', 'star_wars', 'playlist', 'onde_voce_mora', 'triste', 'historia', 'piada', 'bff', 'esporte', 'me', 'cor', 'risada', 'time', 'genero', 'comida'\n", + "\t- Found intents: 'genero', 'filme', 'filhos', 'onde_voce_mora', 'piada', 'star_wars', 'historia', 'esporte', 'como_estou', 'risada', 'comida', 'bff', 'time', 'me', 'license', 'relationship', 'signo', 'cor', 'religiao', 'de_onde_voce_eh', 'linguagens', 'hobby', 'playlist', 'triste'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'license', 'triste', 'starwars', 'genero', 'linguagens', 'comida', 'religiao', 'live', 'filme', 'hobby', 'signo', 'relationship', 'playlist', 'piada', 'bff', 'how', 'esporte', 'where', 'me', 'cor', 'filhos', 'historia'\n", + "\t- found entities: 'filme', 'genero', 'filhos', 'live', 'me', 'cor', 'hobby', 'where', 'triste', 'starwars', 'piada', 'how', 'esporte', 'comida', 'bff', 'license', 'relationship', 'signo', 'religiao', 'linguagens', 'playlist', 'historia'\n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/actions.md is md\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/actions.md is md\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 6 (1 distinct intents)\n", "\t- Found intents: 'action_test'\n", "\t- entity examples: 0 (0 distinct entities)\n", "\t- found entities: \n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.loading\u001b[0m - Training data format of data/intents/geral.md is md\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", - "\t- intent examples: 85 (7 distinct intents)\n", - "\t- Found intents: 'despedir', 'diga_mais', 'out_of_scope', 'elogios', 'cumprimentar', 'negar', 'tudo_bem'\n", - "\t- entity examples: 0 (0 distinct entities)\n", - "\t- found entities: \n", - "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.training_data.training_data\u001b[0m - Training data stats: \n", "\t- intent examples: 288 (32 distinct intents)\n", - "\t- Found intents: 'license', 'como_estou', 'star_wars', 'negar', 'elogios', 'onde_voce_mora', 'triste', 'despedir', 'diga_mais', 'cumprimentar', 'genero', 'comida', 'linguagens', 'religiao', 'filme', 'hobby', 'de_onde_voce_eh', 'signo', 'relationship', 'out_of_scope', 'playlist', 'tudo_bem', 'piada', 'bff', 'risada', 'esporte', 'me', 'action_test', 'cor', 'filhos', 'time', 'historia'\n", + "\t- Found intents: 'genero', 'filme', 'filhos', 'elogios', 'out_of_scope', 'tudo_bem', 'me', 'despedir', 'cor', 'de_onde_voce_eh', 'hobby', 'triste', 'onde_voce_mora', 'piada', 'star_wars', 'esporte', 'como_estou', 'risada', 'comida', 'negar', 'bff', 'time', 'action_test', 'diga_mais', 'license', 'relationship', 'cumprimentar', 'signo', 'religiao', 'linguagens', 'playlist', 'historia'\n", "\t- entity examples: 172 (22 distinct entities)\n", - "\t- found entities: 'license', 'triste', 'starwars', 'genero', 'linguagens', 'comida', 'religiao', 'live', 'filme', 'hobby', 'signo', 'relationship', 'playlist', 'piada', 'bff', 'how', 'esporte', 'where', 'me', 'cor', 'filhos', 'historia'\n", + "\t- found entities: 'filme', 'genero', 'filhos', 'live', 'me', 'cor', 'hobby', 'where', 'triste', 'starwars', 'piada', 'how', 'esporte', 'comida', 'bff', 'license', 'relationship', 'signo', 'religiao', 'linguagens', 'playlist', 'historia'\n", "\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:47 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", - "2019-05-03 21:20:48 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:20:48 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component WhitespaceTokenizer\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:34:21 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CRFEntityExtractor\n", + "2019-07-17 16:34:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:34:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EntitySynonymMapper\n", + "2019-07-17 16:34:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:34:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component CountVectorsFeaturizer\n", + "2019-07-17 16:34:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:34:22 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Starting to train component EmbeddingIntentClassifier\n", "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", "For more information, please see:\n", @@ -222,50 +210,48 @@ " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n", - "2019-05-03 21:20:48 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:23 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:285: dense (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use keras.layers.dense instead.\u001b[0m\n", - "2019-05-03 21:20:48 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:23 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\u001b[0m\n", - "2019-05-03 21:20:48 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:23 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/rasa_nlu/classifiers/embedding_intent_classifier.py:286: dropout (from tensorflow.python.layers.core) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use keras.layers.dropout instead.\u001b[0m\n", - "2019-05-03 21:20:48 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:23 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/layers/core.py:143: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", - "2019-05-03 21:20:48 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:23 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use tf.cast instead.\u001b[0m\n", - "2019-05-03 21:20:48 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:23 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n", - "2019-05-03 21:20:49.119991: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-05-03 21:20:49.139740: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2905000000 Hz\n", - "2019-05-03 21:20:49.140045: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x555c5d1fc6c0 executing computations on platform Host. Devices:\n", - "2019-05-03 21:20:49.140072: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-05-03 21:20:49 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", - "Epochs: 100%|██████████| 300/300 [00:11<00:00, 26.20it/s, loss=0.105, acc=0.993]\n", - "2019-05-03 21:21:00 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.105, train accuracy=0.993\n", - "2019-05-03 21:21:00 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", - "2019-05-03 21:21:00 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Successfully saved model into '/work/bot/models/nlu/current'\n", - "2019-05-03 21:21:00 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished training\n", + "2019-07-17 16:34:24.429290: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:34:24.452899: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:34:24.453405: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x56418a533710 executing computations on platform Host. Devices:\n", + "2019-07-17 16:34:24.453460: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:34:24 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Accuracy is updated every 10 epochs\n", + "Epochs: 100%|██████████| 300/300 [00:13<00:00, 22.67it/s, loss=0.100, acc=0.993]\n", + "2019-07-17 16:34:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.classifiers.embedding_intent_classifier\u001b[0m - Finished training embedding classifier, loss=0.100, train accuracy=0.993\n", + "2019-07-17 16:34:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Finished training component.\n", + "2019-07-17 16:34:38 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_nlu.model\u001b[0m - Successfully saved model into '/src_models/nlu/current'\n", + "2019-07-17 16:34:38 \u001b[1;30mINFO \u001b[0m \u001b[34m__main__\u001b[0m - Finished training\n", "python3 train.py\n", - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", - "/work/bot/validator.py:98: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n", - " validate(yaml.load(domain_file), yaml.load(schema))\n" + "/work/coach/validator.py:98: YAMLLoadWarning: calling yaml.load() without Loader=... is deprecated, as the default Loader is unsafe. Please read https://msg.pyyaml.org/load for full details.\n", + " validate(yaml.load(domain_file), yaml.load(schema))\n", + "2019-07-17 16:34:39 \u001b[1;30mINFO \u001b[0m \u001b[34mvalidator\u001b[0m - Domain verified\n", + "2019-07-17 16:34:39 \u001b[1;30mERROR \u001b[0m \u001b[34mvalidator\u001b[0m - \u001b[31mThe utter action_test is used in the stories story file data/stories/actions.md (line: 5) but it's not a valid utter.\u001b[0m\n", + "2019-07-17 16:34:41 \u001b[1;30mINFO \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Scheduler started\n", + "2019-07-17 16:34:41 \u001b[1;30mDEBUG \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Looking for jobs to run\n", + "2019-07-17 16:34:41 \u001b[1;30mDEBUG \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - No jobs; waiting until a job is added\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2019-05-03 21:21:02 \u001b[1;30mINFO \u001b[0m \u001b[34mvalidator\u001b[0m - Domain verified\n", - "2019-05-03 21:21:02 \u001b[1;30mERROR \u001b[0m \u001b[34mvalidator\u001b[0m - \u001b[31mThe utter action_test is used in the stories story file data/stories/actions.md (line: 5) but it's not a valid utter.\u001b[0m\n", - "2019-05-03 21:21:04 \u001b[1;30mINFO \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Scheduler started\n", - "2019-05-03 21:21:04 \u001b[1;30mDEBUG \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - Looking for jobs to run\n", - "2019-05-03 21:21:04 \u001b[1;30mDEBUG \u001b[0m \u001b[34mapscheduler.scheduler\u001b[0m - No jobs; waiting until a job is added\n", "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", "For more information, please see:\n", @@ -273,36 +259,36 @@ " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Number of augmentation rounds is 3\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting data generation round 0 ... (with 1 trackers)\n", - "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 5591.53it/s, # trackers=1]\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (34 training samples found).\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Data generation rounds finished.\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Found 0 unused checkpoints\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 0 ... (with 20 trackers)\n", - "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 426.94it/s, # trackers=20]\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (711 training samples found).\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 1 ... (with 20 trackers)\n", - "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 300.58it/s, # trackers=20]\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (1391 training samples found).\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 2 ... (with 20 trackers)\n", - "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 174.15it/s, # trackers=20]\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (2034 training samples found).\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Found 2034 training trackers.\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Subsampled to 200 augmented training trackers.\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - There are 34 original trackers.\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Agent trainer got kwargs: {'validation_split': 0.2}\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Creating states and action examples from collected trackers (by FullDialogueTrackerFeaturizer(LabelTokenizerSingleStateFeaturizer))...\n", - "Processed trackers: 100%|██████████████████| 234/234 [00:00<00:00, 19008.16it/s]\n", - "2019-05-03 21:21:05 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - The longest dialogue has 13 actions.\n", - "2019-05-03 21:21:05.665574: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-05-03 21:21:05.686532: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2905000000 Hz\n", - "2019-05-03 21:21:05.688378: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55cad90cbdc0 executing computations on platform Host. Devices:\n", - "2019-05-03 21:21:05.688446: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", - "2019-05-03 21:21:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Number of augmentation rounds is 3\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting data generation round 0 ... (with 1 trackers)\n", + "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 4790.91it/s, # trackers=1]\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (34 training samples found).\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Data generation rounds finished.\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Found 0 unused checkpoints\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 0 ... (with 34 trackers)\n", + "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 204.51it/s, # trackers=34]\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (1187 training samples found).\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 1 ... (with 50 trackers)\n", + "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 111.16it/s, # trackers=50]\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (2819 training samples found).\n", + "2019-07-17 16:34:42 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Starting augmentation round 2 ... (with 50 trackers)\n", + "Processed Story Blocks: 100%|████| 34/34 [00:00<00:00, 94.44it/s, # trackers=50]\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Finished phase (4439 training samples found).\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Found 4439 training trackers.\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - Subsampled to 500 augmented training trackers.\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.training.generator\u001b[0m - There are 34 original trackers.\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Agent trainer got kwargs: {'validation_split': 0.2}\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Creating states and action examples from collected trackers (by FullDialogueTrackerFeaturizer(LabelTokenizerSingleStateFeaturizer))...\n", + "Processed trackers: 100%|██████████████████| 534/534 [00:00<00:00, 29542.81it/s]\n", + "2019-07-17 16:34:43 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - The longest dialogue has 11 actions.\n", + "2019-07-17 16:34:43.394363: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:34:43.420891: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:34:43.421344: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55f4cd084a60 executing computations on platform Host. Devices:\n", + "2019-07-17 16:34:43.421423: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:34:43 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\u001b[0m\n", - "2019-05-03 21:21:05 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:43 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/keras/backend.py:4010: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\u001b[0m\n", "_________________________________________________________________\n", @@ -320,51 +306,74 @@ "Trainable params: 18,183\n", "Non-trainable params: 0\n", "_________________________________________________________________\n", - "2019-05-03 21:21:06 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - None\n", - "2019-05-03 21:21:06 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - Fitting model with 234 total samples and a validation split of 0.1\n", - "2019-05-03 21:21:06 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.policy\u001b[0m - Parameters ignored by `model.fit(...)`: {}\n", - "2019-05-03 21:21:06 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:44 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - None\n", + "2019-07-17 16:34:44 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - Fitting model with 534 total samples and a validation split of 0.1\n", + "2019-07-17 16:34:44 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.policy\u001b[0m - Parameters ignored by `model.fit(...)`: {}\n", + "2019-07-17 16:34:44 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Use tf.cast instead.\u001b[0m\n", - "2019-05-03 21:21:06 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", + "2019-07-17 16:34:44 \u001b[1;30mWARNING \u001b[0m \u001b[34mtensorflow\u001b[0m - \u001b[33mFrom /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Deprecated in favor of operator or tf.math.divide.\u001b[0m\n", - "Epoch 1/8\n", - "234/234 [==============================] - 2s 7ms/sample - loss: 3.4823 - acc: 0.4455\n", - "Epoch 2/8\n", - "234/234 [==============================] - 0s 2ms/sample - loss: 2.9316 - acc: 0.5708\n", - "Epoch 3/8\n", - "234/234 [==============================] - 0s 2ms/sample - loss: 2.6178 - acc: 0.5708\n", - "Epoch 4/8\n", - "234/234 [==============================] - 1s 2ms/sample - loss: 2.4668 - acc: 0.5708\n", - "Epoch 5/8\n", - "234/234 [==============================] - 1s 2ms/sample - loss: 2.3236 - acc: 0.5708\n", - "Epoch 6/8\n", - "234/234 [==============================] - 1s 2ms/sample - loss: 2.1960 - acc: 0.5708\n", - "Epoch 7/8\n", - "234/234 [==============================] - 1s 2ms/sample - loss: 2.0711 - acc: 0.5708\n", - "Epoch 8/8\n", - "234/234 [==============================] - 1s 2ms/sample - loss: 1.9513 - acc: 0.5708\n", - "2019-05-03 21:21:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - Done fitting keras policy model\n", - "2019-05-03 21:21:13 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Creating states and action examples from collected trackers (by MaxHistoryTrackerFeaturizer(SingleStateFeaturizer))...\n", - "Processed trackers: 100%|███████| 34/34 [00:00<00:00, 4771.84it/s, # actions=68]\n", - "2019-05-03 21:21:13 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Created 68 action examples.\n", - "Processed actions: 68it [00:00, 6188.17it/s, # examples=68]\n", - "2019-05-03 21:21:13 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.memoization\u001b[0m - Memorized 68 unique examples.\n", - "2019-05-03 21:21:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Model directory models/dialogue exists and contains old model files. All files will be overwritten.\n" + "Epoch 1/20\n", + "534/534 [==============================] - 2s 3ms/sample - loss: 3.1508 - acc: 0.5214\n", + "Epoch 2/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 2.5611 - acc: 0.5706\n", + "Epoch 3/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 2.2887 - acc: 0.5706\n", + "Epoch 4/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 2.0275 - acc: 0.5706\n", + "Epoch 5/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 1.8310 - acc: 0.5915\n", + "Epoch 6/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 1.7020 - acc: 0.6172\n", + "Epoch 7/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 1.5782 - acc: 0.6386\n", + "Epoch 8/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 1.4444 - acc: 0.6706\n", + "Epoch 9/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 1.2862 - acc: 0.7177\n", + "Epoch 10/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 1.1180 - acc: 0.7656\n", + "Epoch 11/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 0.9341 - acc: 0.8222\n", + "Epoch 12/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 0.7627 - acc: 0.8746\n", + "Epoch 13/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 0.6112 - acc: 0.9196\n", + "Epoch 14/20\n", + "534/534 [==============================] - 1s 2ms/sample - loss: 0.4943 - acc: 0.9331\n", + "Epoch 15/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 0.4015 - acc: 0.9474\n", + "Epoch 16/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 0.3283 - acc: 0.9558\n", + "Epoch 17/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 0.2693 - acc: 0.9640\n", + "Epoch 18/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 0.2484 - acc: 0.9622\n", + "Epoch 19/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 0.2007 - acc: 0.9680\n", + "Epoch 20/20\n", + "534/534 [==============================] - 1s 1ms/sample - loss: 0.1921 - acc: 0.9698\n", + "2019-07-17 16:35:03 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.policies.keras_policy\u001b[0m - Done fitting keras policy model\n", + "2019-07-17 16:35:03 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Creating states and action examples from collected trackers (by MaxHistoryTrackerFeaturizer(SingleStateFeaturizer))...\n", + "Processed trackers: 100%|███████| 34/34 [00:00<00:00, 4103.78it/s, # actions=68]\n", + "2019-07-17 16:35:03 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.featurizers\u001b[0m - Created 68 action examples.\n", + "Processed actions: 68it [00:00, 9631.33it/s, # examples=68]\n", + "2019-07-17 16:35:03 \u001b[1;30mDEBUG \u001b[0m \u001b[34mrasa_core.policies.memoization\u001b[0m - Memorized 68 unique examples.\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "2019-05-03 21:21:13 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Persisted model to '/work/bot/models/dialogue'\n", - "make: Leaving directory '/work/bot'\n" + "2019-07-17 16:35:04 \u001b[1;30mINFO \u001b[0m \u001b[34mrasa_core.agent\u001b[0m - Persisted model to '/src_models/dialogue'\n", + "make: Leaving directory '/work/coach'\n" ] } ], "source": [ - "!make train -C $BOT_DIR_PATH" + "!make train -C $COACH_DIR_PATH" ] }, { @@ -385,8 +394,6 @@ "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/lib/python3.6/site-packages/requests/__init__.py:91: RequestsDependencyWarning: urllib3 (1.25.2) or chardet (3.0.4) doesn't match a supported version!\n", - " RequestsDependencyWarning)\n", "Calling `rasa_core.evaluate` is deprecated. Please use `rasa_core.test` instead.\n", "\n", "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n", @@ -395,10 +402,10 @@ " * https://github.com/tensorflow/addons\n", "If you depend on functionality not listed there, please file an issue.\n", "\n", - "2019-05-03 21:21:17.252292: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", - "2019-05-03 21:21:17.270392: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2905000000 Hz\n", - "2019-05-03 21:21:17.271557: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x55efe976d490 executing computations on platform Host. Devices:\n", - "2019-05-03 21:21:17.271656: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", + "2019-07-17 16:35:22.074432: I tensorflow/core/platform/cpu_feature_guard.cc:141] Your CPU supports instructions that this TensorFlow binary was not compiled to use: AVX2 FMA\n", + "2019-07-17 16:35:22.096944: I tensorflow/core/platform/profile_utils/cpu_utils.cc:94] CPU Frequency: 2194805000 Hz\n", + "2019-07-17 16:35:22.098072: I tensorflow/compiler/xla/service/service.cc:150] XLA service 0x5576fe7115f0 executing computations on platform Host. Devices:\n", + "2019-07-17 16:35:22.098197: I tensorflow/compiler/xla/service/service.cc:158] StreamExecutor device (0): , \n", "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/resource_variable_ops.py:435: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Colocations handled automatically by placer.\n", @@ -423,10 +430,10 @@ "WARNING:tensorflow:From /usr/local/lib/python3.6/site-packages/tensorflow/python/ops/math_grad.py:102: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n", "Instructions for updating:\n", "Deprecated in favor of operator or tf.math.divide.\n", - "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 6296.36it/s, # trackers=1]\n", + "Processed Story Blocks: 100%|███| 34/34 [00:00<00:00, 4384.65it/s, # trackers=1]\n", "INFO:rasa_core.test:Evaluating 34 stories\n", "Progress:\n", - "100%|███████████████████████████████████████████| 34/34 [00:00<00:00, 68.13it/s]\n", + "100%|███████████████████████████████████████████| 34/34 [00:00<00:00, 55.92it/s]\n", "INFO:rasa_core.test:Finished collecting predictions.\n", "INFO:rasa_core.test:Evaluation Results on CONVERSATION level:\n", "INFO:rasa_core.test:\tCorrect: 34 / 34\n", @@ -493,7 +500,7 @@ } ], "source": [ - "!python -m rasa_core.evaluate --core $BOT_MODELS_DIALOGUE_PATH -s $BOT_STORIES_PATH -o results" + "!python -m rasa_core.evaluate --core $COACH_MODELS_DIALOGUE_PATH -s $COACH_STORIES_PATH -o results" ] }, { @@ -538,7 +545,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -555,10 +562,10 @@ " " ], "text/plain": [ - "" + "" ] }, - "execution_count": 7, + "execution_count": 11, "metadata": {}, "output_type": "execute_result" } @@ -623,7 +630,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.8" + "version": "3.6.9" } }, "nbformat": 4,