From a3b825ee86301bec7bbe429156f438ab38385a68 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Wed, 4 Oct 2023 10:21:28 +0100 Subject: [PATCH 01/18] initial nlp template project code --- .github/actions/nlp_template_test/action.yml | 85 ++++++++ .github/workflows/ci.yml | 37 ++++ .gitignore | 164 ++++++++++++++ .pytest.ini | 5 + LICENSE | 201 ++++++++++++++++++ README.md | 1 + RELEASE_NOTES.md | 3 + copier.yml | 147 +++++++++++++ requirements.txt | 2 + template/.assets/00_pipelines_composition.png | Bin 0 -> 77711 bytes template/.dockerignore | 2 + template/Makefile | 24 +++ template/README.md | 73 +++++++ template/artifacts/__init__.py | 1 + template/artifacts/materializer.py | 89 ++++++++ template/artifacts/model_metadata.py | 62 ++++++ template/config.py | 80 +++++++ template/license | 110 ++++++++++ template/license_header | 2 + template/pipelines/__init__.py | 4 + template/pipelines/training.py | 115 ++++++++++ template/requirements.txt | 1 + template/run.py | 169 +++++++++++++++ template/steps/__init__.py | 19 ++ template/steps/alerts/__init__.py | 4 + template/steps/alerts/notify_on.py | 42 ++++ template/steps/dataset_loader/__init__.py | 4 + template/steps/dataset_loader/data_loader.py | 61 ++++++ template/steps/inference/__init__.py | 5 + .../inference_get_current_version.py | 34 +++ template/steps/inference/inference_predict.py | 43 ++++ template/steps/promotion/__init__.py | 5 + .../steps/promotion/promote_get_versions.py | 50 +++++ template/steps/promotion/promote_latest.py | 49 +++++ template/steps/tokenizer_loader/__init__.py | 4 + .../tokenizer_loader/tokenizer_loader.py | 53 +++++ template/steps/tokenzation/__init__.py | 4 + template/steps/tokenzation/tokenization.py | 71 +++++++ template/steps/training/__init__.py | 3 + template/steps/training/model_trainer.py | 123 +++++++++++ template/utils/misc.py | 39 ++++ ...f open_source_license %}LICENSE{% endif %} | 1 + template/{{_copier_conf.answers_file}} | 2 + test-requirements.txt | 6 + 44 files changed, 1999 insertions(+) create mode 100644 .github/actions/nlp_template_test/action.yml create mode 100644 .github/workflows/ci.yml create mode 100644 .gitignore create mode 100644 .pytest.ini create mode 100644 LICENSE create mode 100644 README.md create mode 100644 RELEASE_NOTES.md create mode 100644 copier.yml create mode 100644 requirements.txt create mode 100644 template/.assets/00_pipelines_composition.png create mode 100644 template/.dockerignore create mode 100644 template/Makefile create mode 100644 template/README.md create mode 100644 template/artifacts/__init__.py create mode 100644 template/artifacts/materializer.py create mode 100644 template/artifacts/model_metadata.py create mode 100644 template/config.py create mode 100644 template/license create mode 100644 template/license_header create mode 100644 template/pipelines/__init__.py create mode 100644 template/pipelines/training.py create mode 100644 template/requirements.txt create mode 100644 template/run.py create mode 100644 template/steps/__init__.py create mode 100644 template/steps/alerts/__init__.py create mode 100644 template/steps/alerts/notify_on.py create mode 100644 template/steps/dataset_loader/__init__.py create mode 100644 template/steps/dataset_loader/data_loader.py create mode 100644 template/steps/inference/__init__.py create mode 100644 template/steps/inference/inference_get_current_version.py create mode 100644 template/steps/inference/inference_predict.py create mode 100644 template/steps/promotion/__init__.py create mode 100644 template/steps/promotion/promote_get_versions.py create mode 100644 template/steps/promotion/promote_latest.py create mode 100644 template/steps/tokenizer_loader/__init__.py create mode 100644 template/steps/tokenizer_loader/tokenizer_loader.py create mode 100644 template/steps/tokenzation/__init__.py create mode 100644 template/steps/tokenzation/tokenization.py create mode 100644 template/steps/training/__init__.py create mode 100644 template/steps/training/model_trainer.py create mode 100644 template/utils/misc.py create mode 100644 template/{% if open_source_license %}LICENSE{% endif %} create mode 100644 template/{{_copier_conf.answers_file}} create mode 100644 test-requirements.txt diff --git a/.github/actions/nlp_template_test/action.yml b/.github/actions/nlp_template_test/action.yml new file mode 100644 index 0000000..8b78d56 --- /dev/null +++ b/.github/actions/nlp_template_test/action.yml @@ -0,0 +1,85 @@ +name: 'Run NLP template tests' +inputs: + stack-name: + description: 'Name of ZenML stack to build (see `tests/conftest.py:configure_stack()`)' + type: string + required: true + ref-zenml: + description: 'Ref of ZenML package' + type: string + required: false + default: '' + ref-template: + description: 'Ref of this template repo' + type: string + required: false + default: '' + python-version: + description: 'Python version' + type: string + required: false + default: '3.9' + +runs: + using: "composite" + steps: + - name: Check out repository code + uses: actions/checkout@v3 + with: + repository: zenml-io/nlp-template + ref: ${{ inputs.ref-template }} + path: ./local_checkout + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: ${{ inputs.python-version }} + + - name: Configure git (non-Windows) + if: ${{ runner.os != 'Windows' }} + shell: bash + run: | + git config --global user.email "info@zenml.io" + git config --global user.name "ZenML GmbH" + + - name: Configure git (Windows) + if: ${{ runner.os == 'Windows' }} + shell: bash + run: | + "C:\Program Files\Git\bin\git.exe" config --global user.email "info@zenml.io" + "C:\Program Files\Git\bin\git.exe" config --global user.name "ZenML GmbH" + + - name: Install wheel + shell: bash + run: | + pip install wheel + + - name: Install ZenML + if: ${{ inputs.ref-zenml != '' }} + shell: bash + run: | + pip install "git+https://github.com/zenml-io/zenml.git@${{ inputs.ref-zenml }}" "zenml[server]@git+https://github.com/zenml-io/zenml.git@${{ inputs.ref-zenml }}" + + - name: Install ZenML + if: ${{ inputs.ref-zenml == '' }} + shell: bash + run: | + pip install zenml "zenml[server]" + + - name: Concatenate requirements + shell: bash + run: | + zenml integration export-requirements -o ./local_checkout/integration-requirements.txt sklearn mlflow s3 kubernetes kubeflow slack evidently + cat ./local_checkout/requirements.txt ./local_checkout/test-requirements.txt ./local_checkout/integration-requirements.txt >> ./local_checkout/all-requirements.txt + + - name: Install requirements + shell: bash + run: | + pip install -r ./local_checkout/all-requirements.txt + + - name: Run pytests + shell: bash + env: + ZENML_STACK_NAME: ${{ inputs.stack-name }} + run: | + pytest ./local_checkout/tests diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..8b0da42 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,37 @@ +name: CI + +on: + workflow_dispatch: + workflow_call: + push: + branches: ["main", "develop"] + paths-ignore: ["README.md"] + pull_request: + paths-ignore: ["README.md"] + +concurrency: + # New commit on branch cancels running workflows of the same branch + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + run-tests: + runs-on: ${{ matrix.os }} + strategy: + matrix: + stack-name: [local] + os: [windows-latest, ubuntu-latest, macos-latest] + python-version: ["3.8", "3.9", "3.10", "3.11"] + env: + ZENML_DEBUG: true + ZENML_ANALYTICS_OPT_IN: false + ZENML_LOGGING_VERBOSITY: INFO + steps: + - name: Check out repository code + uses: actions/checkout@v3 + + - name: Run tests + uses: ./.github/actions/nlp_template_test + with: + stack-name: ${{ matrix.stack-name }} + python-version: ${{ matrix.python-version }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c1b8c59 --- /dev/null +++ b/.gitignore @@ -0,0 +1,164 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv* +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +.vscode/ +.local/ +.ruff_cache/ \ No newline at end of file diff --git a/.pytest.ini b/.pytest.ini new file mode 100644 index 0000000..4c1c80e --- /dev/null +++ b/.pytest.ini @@ -0,0 +1,5 @@ +[pytest] +addopts = + -s +testpaths = + tests \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..448b80b --- /dev/null +++ b/README.md @@ -0,0 +1 @@ +# 💫 ZenML End-to-End Natural Language Processing Project Template \ No newline at end of file diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..92ad829 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,3 @@ + + + diff --git a/copier.yml b/copier.yml new file mode 100644 index 0000000..be5ab4c --- /dev/null +++ b/copier.yml @@ -0,0 +1,147 @@ +# Copyright (c) ZenML GmbH 2023. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing +# permissions and limitations under the License. + + +# GLOBAL PROMPT -------------------------------- +project_name: + type: str + help: Short name for your project + default: ZenML NLP project +version: + type: str + help: | + Version of your project + default: "0.0.1" +open_source_license: + type: str + help: >- + The license under which your project will be released + choices: + Apache Software License 2.0: apache + MIT license: mit + BSD license: bsd + ISC license: isc + GNU General Public License v3: gpl3 + Not open source: none + default: apache +full_name: + type: str + help: >- + The name of the person/entity holding the copyright + default: ZenML GmbH + when: "{{ open_source_license != 'none' }}" +email: + type: str + help: >- + The email of the person/entity holding the copyright + default: info@zenml.io + when: "{{ open_source_license != 'none' }}" +product_name: + type: str + help: The technical name of the data product you are building + default: nlp_use_case +target_environment: + type: str + help: "The target environment for your project" + choices: + - production + - staging + default: staging +dataset: + type: str + help: "The name of the dataset to use from HuggingFace Datasets" + choices: + - financial_news + - airline_reviews + - imbd_reviews + default: financial_news +model: + type: str + help: "The name of the model to use from HuggingFace Models" + choices: + - bert + - gpt2 + default: bert +cloud_of_choice: + type: str + help: "Whether to use AWS cloud provider or GCP" + choices: + - aws + - gcp + default: aws +notify_on_failures: + type: bool + help: "Whether to notify on pipeline failures?" + default: True +notify_on_successes: + type: bool + help: "Whether to notify on pipeline successes?" + default: False +zenml_server_url: + type: str + help: "The URL of the ZenML server [Optional]" + default: "" + + +# CONFIGURATION ------------------------- +_templates_suffix: "" +_subdirectory: template +_exclude: + - license + - license_header + +_tasks: + # Remove unused imports and variables + - >- + {% if _copier_conf.os == 'windows' %} + echo "Auto-formatting not supported on Windows" + {% else %} + {{ _copier_python }} -m ruff --select F401,F841 --fix \ + --exclude "__init__.py" --isolated \ + steps pipelines run.py > /dev/null 2>&1 || true + {% endif %} + # Sort imports + - >- + {% if _copier_conf.os == 'windows' %} + echo "Auto-formatting not supported on Windows" + {% else %} + {{ _copier_python }} -m ruff --select I \ + --fix --ignore D \ + steps pipelines run.py > /dev/null 2>&1 || true + {% endif %} + # Auto-format code + - >- + {% if _copier_conf.os == 'windows' %} + echo "Auto-formatting not supported on Windows" + {% else %} + {{ _copier_python }} -m black \ + --exclude '' --include '\.pyi?$' -l 79 \ + steps pipelines run.py > /dev/null 2>&1 || true + {% endif %} + - | + echo "Congratulations, your project has been generated in the '{{ _copier_conf.dst_path }}' directory." + echo "You can now run the following commands to get started:" + echo " cd {{ _copier_conf.dst_path }}" + echo " make setup" + echo " # optional, provision default local stack" + echo " make install-stack" + echo " # optional, start ZenML Dashboard" + echo " zenml up" + echo " python run.py" + echo + echo "Next, you should take a look at the '{{ _copier_conf.dst_path }}/README.md' file in the generated project." + echo "Happy coding!" + +_jinja_extensions: + - jinja2_time.TimeExtension diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3f1435b --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +copier +jinja2-time diff --git a/template/.assets/00_pipelines_composition.png b/template/.assets/00_pipelines_composition.png new file mode 100644 index 0000000000000000000000000000000000000000..bd614886baa43641d1224b0dab9d9fcb6b75019c GIT binary patch literal 77711 zcmeFZcTiK^_ct0uMFl}cg+~yqfT9!yL8&5&NVOtDh=7lXNRtvu2v`8=C@M%-1f*l3 zC6LgIC?LHhL3&FTKPz zXA1-Z*{XX{`wIA?Kp-1DHf;n~P~Gk&;A^wfMN=0DgjbOL!v%?r7X}x(T(0P7L2{c! zC%_-;ZO$5;g+M+=@UB>HfN;qk($zkD&4X(i2YY7TlOR5m^2xok!nyI72h|e2|IpL? z$w%dwY%l#!4ON-{*2i_I)eZ!{@|t~79)1fcf9OWU=HZ$bclQhIKX~NPwyKN0f!m&> z7kJi_tctYuMJ~9?n$|mX;*c>*`?JJOIoIk*pHalbwxyGEG@w3aR52fQ<7*u`-i`^G zt$C*pHx(gm64e3a93MYHrA-`vLLi#bPnZ6>kbGDR@z;fx9sAY)x^OROoA_TBzGy!a z`0K*01Jyi#T{ymdkel;@-|I~jF32{HOZNi*4^7H3r>5TjcYFIEGJbL3m!$uP8~?)% z2;~3T3;d66{Qprm-lQ8Pb>ba*(@`*J?4me47d2+k+~l4A5K3# zQWq{_<^^MPX+$rZu%bw?A}AxvcOj3N?%dyeM8mzmyOlnPWtOyKmy=vsDB>iHeOn`J z2x&>7*FqtF{>wpIHgWEbs*k{`A}m(#t#Eb}?a8w4*RetkJnv6TIIWZRJbfj$bgR5U%nOY&9qHuACq@r zeV8Z{U+DhOJfh6n*!vP$`$K`n-@&5PDm7r*L{peXOyLC>7S?uZL4&p=(F2dQZf7-! zr!c3ecD-p#NpZpUzX+IQ z4M$vNEzlM1ud<>d9AEgPpT59IT9Kc`BC%=qlfJVQgHL3Ix(}=j%!FLv1UuMJ>X0RIS{+|%1j}-CSrd~kNhj?sO^wEv0{b1c%7wgbGSUfgO*Ypk)paCTd9aWldSm+7G>-jt1amqgZgAy+}ChHgK@gu zSL4I~i^-QJ1nj}ALd-}jDk9el%T)m{z}iBgBP4Hr%b z9dGT54}UvhPP4B22gbam6K2`&qLkngYh7obH1hGX^~JC=$o{d@q?pS^b!seyyc|;h zXXce}&Pf`$*Adn)s4^O9q?(ox&=Q{#^X#@M6^lhhguQ#*->GKem3x`o_8x{$qrEsWNw|h_#Q3(|to5Pnaci>0Y%`?4>HKGbW;j>l zbR{YtJF&ptpo54aDb3}5^DA}ivIZ(?*-y0Hy*)f_FSY2e%O zj0s8&t46f@F0RM6)~(=nTvIxD(#zr3lg$y=S(ei!twb8p7G}r1$!3-92j2yPt2%g~ z&1Kb}boe5(mj$ewUpX{88qsfM*&EPPsZ)zwQa$OX5;sFOG)g}2&~|RW^Vwnc4L+i1 zs_EqSm`O~`DTCN7qCphtaugmlX4tGkikbA3cB3n)`QoNT-4fHrjPmAO(#d+bA%kW! zs%Kr}Z8_yxQ*oWPF=bK=f`y_Y+oNlgnYQ>o6!kb(jf9xMrl4fqY1p^AnD=4JRz0ra zY1HD0IZ>+PopvD9A1;}!(J{6wYM*2O#-2WlRaM7!dZd!JxlF)9$3{o+C+spS>N0B* z8w3=_pl+D{dzWY2e=#Lb#R9Owxj7-x1bidRkY?a3csVJQ{W$QZj%OqYtG?9NwVzfp zH)r65^iyvQ5|Fy3-mq8kjpG^huaaMBM3nBoNW_^$#Df%x;S@}Ds23AOxgn9Tq~v3p zDPq^wJv;V+J}Y`1mKcYRkVt95%_H1~a=cZ26EBl{-Z!6Yh)%oQtT9Qri1SvQBvij= zSxh#)Qg_7J)i*1W2@!m>f2MD9)sEBinz;o;g2(I5g4Aqt8 zx#hhLS{8WgkUhzoxjAKV4$0Q8wz=UUW7f|WbznA1drwX-Ig{;hcK6;f&wUWd0@~P# z>WWO#1N@p-r75@jDu0Ps<#8ymbowpop4bnpZIiW`|2Ig4eqc#VSX10_@}i8m94E8S@g9ykoGM1pp*e?ESqSZA*Zv2Ffv_CzD0SaxU1Vq=nq zFVMXEAbR@f#x6E$w*ihu)X54{^sG@ZoVPI?lh$>$NX@r&!mYDRDM~a6-&_YRDr@-Q z>pg@INu%h`#t-2G5&A>w^fQyJ@$Oa8uLe69V{8a05IbU~S%b^XV9Vmj|C)!bU2Nx4 zH~fG8*>ot&U`(&*`q}bg`~ES$78@;u=JQq!ds!tM9`qtHiqYmtmzGrdp@8WTCOimrJXNbCG87j^`Chwp*BbFz{{QUduIXsza@$M^ zjPIwT%-*M1hDhMKFoU0JXVsGxY`Dwy-ffwynbJV2ZesoH+_!?o%9U`WzxNt>d>@YX z9Dd4TE?xa8fMF=CB;+A>jE2~u-3Z(k8^Mm9Cs8vsB<~Jcg%uKZ`Pz~!3(1)*D+BFU z7R0#FuQuljROb0EMD2q?mDK%ST>J>BxuilqFJDikvmEx7b7bK&>lhV9JeG5?#D8)r z;j;w|g03 z2jtJubm8z4?aYfp0Qzd z+FT&n#I9_^qTh?n6t44fmK?e;o?iN#SJRg51gaMsC@;RM%yF#^Zc*##B_j-!H8-;# z7=EM{rL=a_kCfGkteYwg3Q3b$ec(X5+W^PZ$@ENqgj>Rr0-Wp8KERQ;TZpNG3THWa zi2T4i!}U@N?D}!Tp0$}ZPiLmd#G)aGw+(U!Un{u6u7~~pBx_{44O~wV+6wdA0ZKUM z>$$zxah8i1R(q9=kM37bzDViXy)wA&KjT25Nn`Uo*WuG5kS*J|y(bPGFWV~~N9P=x z%str&9}6>AaO1l4a5UrI3FBvz%{>(r6?66!_kLy+BZ-r(uFbI1)aa_IiraFfx$Atc ze-?VAo=o1zl`Q8xc=rVd#VxY7w`K9W2e?<&B0V;nlfOP4%jC>C3$an|aTP%HCW*Ui z1VjG4h$c@|^Ssn#^C#$^wL6*`rULvg*~)SW|6JB^RDXQ!9t0%~<58cmhWN0x@UX!2 zn|zK1=j(1eDNnx3;5A;y=Fx<fi+{tlhy}N_Mb+q_Q^pYnF7Y2Um;kCWg*&AOu2D_RRETV7mCgKf? zxaME6?Lqa6%_mM72-w~;Iw!>57RNA1shll$wD3S%%Lb|MJrHBR`RLs%lbkB!``)g6 z!;;?)AcdW8ZQtEp_ZLyX1aZrb1{SNDSd$^);QT4=)rqwQG&R~cEN1iYTyi=E*?GXQ zY4+6D>yeIAaDFm+K6ULeNI7q09+^qU(kO0wy7juA@Xm7iNP-pjZzXqZpoy`Xe(LtM zMt`}+u_2G`vkN%+>gd1)et!M{Dw{^Wb(&nWF!N`bFet~|$r`Q%?n?*3Z0TD_Z@zG< zf)Q%BxwX(QnYIR*is1!EIYc@xV~2Uv%Jvr*CGBbU*iw#3Tc9bO2;L>GRr`!G99Yj; z!CkJ2b^32cyI$z^F_KhYU5ldM%Q4mRei$o9P<&M#H#u8%?(95i%Sna!R;b^V_^lBC zXO?A?S8yEsPu9rh=DGA9jD5Pq%iD37z`r)FCJ?c=IqtMTI}zNe zg1{9ByqWZuIaw*gw{Brp419QrbCwHS|prQd|wf?liLgrjP8fLb8x4I`&cpz@6 zOl1`ApOBD{p`K)$nwM^5HCBtF9u81PCuih_n`x{uNjB>XH%um%S7?%Z6&VOh+O9*S zOS|LEP&58cQ=NT+JWCHL#dNB8z^leQ4KycW@{2xKu+jXcfE3$1i3%3A?Lw1t%$X0n zaX4Pd{fRb=-mZYL4Qp#zICKVbA65pz%GeE`l{=(O92nTwk}Lcr1V^oX6Boxj*7V)K z#3uS4HPG8yWwBE=Df@@7GDrkMJOFR4&78I zFH&;vA+2;Vfp=PtQ|$l=D)Fqpx)6=YoQT z2cbH3MIR|)`Ad_7_0~e8V@ZT?^5q2fj=jU{ejYN1qW9dmVRC^acrL;uuhjVtee(0r zjT^ENs;4l4Ya49&cVf?s!WYT%-{GwsKY^NS!qq}xe4#lo?C#Fa+)&lJl}f}-XtC%G z6Ip@gUFRMZG`o#i+{AoANIg*2oyk8UEX<9l+bUC-Dl;5o=cP8si|sGza=BtftzDRM z6zjnh7Yn0qTO!}OBY1f^V2xkTI9L|+jQldH`MnIdFv1qW2~(qE>AdnedDzp_-9qTT zns65#F|51V12v|_z{m-{r@1g*=&mOppkiwcT;FSIYTmGX;$ISp#QX(DUBX^#gg_C2aQXx$H8s@|UK!OI-Mj{3JZQrSG{`FG z)Y{XX<-kixDDC#oYg%5glia;zYKGoJDs_=PhVfF*&Vz|;q{l${?10kFJ|}u>EPo!0 zsGAw?>>N55QD^+`ifC;Dn$&I17YK!-PdM*8>3-WBSjE9UM;!A8@id9xE8Lj})%bGU z{aL@)O4(Qhq0B|;fg|~;L{?U`B$L`dpIEDDhr{-n6*-7+`EWu0{qjHsx~dP3FX`)^ zm{lGE1sYxTyrb7ds-9fN(c{9`q>#LIosZ^bAOgrl^)Y;|49!{;#<#n}?PtjTjD-cI zD&3%jS~0a{lAQ3Na`>534yB5GDPg>BBUU!GKxvH-5BQpMVDw(On4+R0Aw#{DB9ST^ z%88@pLN&>~{xG^05QDq&ac2a@r@wn?v>pPx7ACuqnWjh_+eHnc*HZ2Ki`Ufp?(;mJ zygjI!y!@-EHZdF_dC;Al*><7D0r~R|YFXZbFAG=VBC9XG@rxE2}c& zB1+bk`rkv~XfbZNI82WM?Cctm}_%qZ(aOmw>8c?iq2@q#Hl**k;oXYFBEQn z$H%sysj05ofD_hm6w1kDrN{}`r6m%{q!oIcbbwi4gn@qw%rXgg$D@5&mfapxwHZ%` zJM&_u&={d){}cuCtU27 zOWz6OL%6=CS(E!ql=wCm;*M|sqa6E!?aEV%nJW0EI5cGb&BYVOFYmv&VbFg-urTT) z8tr|jufL>i!tet6hKWh#6CSc*`*mj@#`mAh4B?F-xPev(OeRq!pnbBGiobi$jBiWr z$1eX3);uU-%}~p#Gj2aIQSNkNTmctGD%^4I2VCUb=eW29GK`Pa;iefnXd~Qwdh%q% zkmvk^B!B1cOrzU7p>lbvkL$`P0m0x-Xb<1my*w%?6iqkXg# z@zCk==-_4-Qh$k4?~NNWMqgl}gj7A9Dpxd`7{F)psVTs~N)m%=@>Ee#DWE&r+jlrb zylu3tZuTOEQTAW_vzDK(Vh%a{tr~z0+H6cS8*~fFB028mUx3mJgZ64SHa1%Bj#>|) zQmIuWl0!Ehzu72Jb#uBsWo59Mo(NMFH|!hS*<7+TNeeo|O(K#y0tBp0DrwfPE#|;a zI^Ym0f7XJ;KSlUZyRdVv-Svmbl_qZEjd7)r`7jXqpMLQ@~fr(u+ z07DD0fkyG)x#KznF;0crv+a|4O9;P>JlfwM#{an+M2NUJtjH-s-#q#4Id{14HMGx9 zVd&D)kTL%?*zV_kZK|TJ+``T5H%i|=YlxWeVQc0W*Pj>tB;76g`ueuc&G9~p*5f1N zP_)I*Q+1D+F)HxogpF&ys5XO_tcc?oPdN*b(iR)ea+qp`pJ+;gW#f9+8&yL;(^2k;pAb2=Kr7EKfD;YW81HFr4>dU`h36?`t+ z%Q1O!pO`Uf)r|>Z?#T&1ee&csYcGendJNCIFCwC1bgO z4QIy#%5Y@>p{$hVT9oaO@p104>Exo!eQ;sn0xhkFF2vl(p2OKBtwBajk-=Ltz~TuK z{X8$V0B3K?mjiVex>IcoO$ZZl^k*;_PCGzwR0h^O#fx3^&qoryy88shCLdEzDcKd( zLB>W?$N>dPx#3%9h$Pa+UK4)@myd?zQWqutDN=&>8t0RbyBR=_c0S==P0B`0gbXeC zHgs%wVilO1WrE-IIXD6DKtTE2Gut)Vr~-Moe2UFup?=9B$I0k1Whd|j-cl~6um3Wa8DHL2YLEm>LCuITPAkPTq2iN#Ql zZ+-&9^YQ{4l^v5wbmW_@$@Od$!YW^n&XaP#XSZeMyh&HCq3F&bSE9z7V_*K!u*UTp zngCru((e+jKXzJ0%5HR=;}4P_R?2R$iEw;?IJq|cm*ZqqG`{P(fK){f1akG38F~k> zh-~8UuBx*_NFpy=iEetjo8G#0U)6|jOAYBkN+lGa;=ty*5G6S2bF=zDRbOEdJmBs3 z%ljLbPQ%J(1-R24Z)!N2Zw45}gzmwIuhqNehJLW6ZH;MP4!h$4I$YnqZ>v0m9zUb|!=`^J#Xc8#bSlVr`qP?J!6U9g~dHQ~Z^rydVLS|yX4aNd@4W#4-p9+U_Rqg*GfZkj^3 z@16`=`bSn4#wP+yIftwvR^rI_Bqt9x*W<3gqT1Sg$B{gAD{MhZa}9ErN;xy&z18PI zEo!GtV(=ku|BJqE`^)y8L$FrFOCR1N*b$H=%z1lzd;eFBF!fJr?nC!animwIg$m*B zl_UXp6*G`tMRyr%@-HcA4xrKe^NT%@9uq06`4>!pr#m2Fy?CjJg+zxpYLsL|Jftl`HdU#V=#-s_j65NIH9h`pr)J3uG|Vr z-uLt%S4KajO-_?Y0U#c+H*|T(8fY=s@-cVaegFrHUB>Wo^+ow|;-|!Y;F&T(MGj?9 zw2!Urku@Uq?OrV4*8M8(AGr3m%ma!ul(g4_ZE3%WT`cG&3W#AuUcZ)EuT{J_+I3$g z@~rrSeh)igydH3wTZ@3#b9v{|kMdOM$Fc_M7lk*@!`BE_4!=Bwp5sTHndLexyxwo! zwWv>E`X)4mtMmatdR-1!010n`TqeJupu=MtPr}QM_qF?zdza(>s1an~y~2it^`Hnh z4p8Vp!=_9_L z@1TltZmtE=-E{NXrX^D|h7v)7$S%Dfcn_t{ZjKE{ls7&Afu)| zKM=rVgVnX6_Vu8MIPWNUAa=3@6!2xbaZr`xEaRPS@>3L^-86dpNtnfW_|?N zRtIBSee2m+Y*JFT#M(UJ=bQwjzQ0L-A)~oJQ^RIr0Tjb$mL=&LLJ$jgkkj&kGJ%*o zKCZJ@#H4C@IZg}nXKZ;8eZK$iS`xeL+p6&|#oZ0|Okf}%#L^i?PTfxu4h@`K!gG3U zIUeK-miEM8lJ}o^H0`XOD(_`E$B$b1io3>=@mm;D!k zNUyBcRoxpn?#!Pn5G&n6teLvxC6b)y3JALWSHa&1WF-~1oj%YLFUV>hVc(B5Y;ql0 zD`G*$qfv?jdrBdzW)f-C)Jk9H!?6HPBA@4_J}B>NiM0#%bB>dJ`@3j4+vH6_mR>7b z+a_eZ5CQeT{Oaf)c%MMcoJn-xEO#@EbGQJol6nq+X#D?H>Mp7ou)y^?g8A8S+<8_v zR##+=Bu#h?B*~4S+0eAnZ{08!6$u|g)6D`UT}^z=0y#T6Li=F^Mg;5oo3XPPg%(LR z?qToajBXYwJ*t=?&cS=gnet_afqzeKg80{?DT?+S`%D(k{#CqPqX1ON1)q4?G{v9( zq~?&-$KRp^$h@2hGo7*H(0mAE+Y>LqeuXbK(Ep^vdB>ItI7(}N%na1*W?&uQNO16T z!`9Zmzj)68P0N=T0nLegm);qX2h1<1br8FJLUz4ib5cax7^~!C)m9UwA*;3TquJ2`v_axuZ40HEg#l+A+$}+4V+$Ox>Eqy??@za`4F-Llfy9k6ni;?z z-o1aT23<=8z`=8)x!uXGVAS(W83YQPlK%5?HM=T_DgM(mke`)7bQJuhCDf5H^TAtA z2GCzEkq^96e|MksP^ZkVf|dhF!5(E6Hgw3!e)Hkb9!LvUTQ<3~dxKQPoAd(>t4`;- zM;=O>MGgAK*;o_Fl^apOH~l}qV8Y?s8hHbo26cJ6iQ!KB~aGT;f8CJQ+T zYq59RO9M`6wJVi8VIq`S=ipoO*YqJ6Wv;RcNYiN*1Gvk9eohh}Z zfIxZZEDVv#%x zlK{W0k=&wY%Y&Ij-4e|G-Ra%rc~dr_cr)iSt|oih*_}^zIC!t@JlFV_8*SOiZU?UR z&l>M=iWjEf%ysDyyV%C+*znhZ2&UlVuK}RZe_c>Pkn^ks1jY86m!KX-CY)O!?I{@j zPaExW=H^>xF1P(9Ky<4jLCgD>i4C6x)tj{~`XN6&_U?Sdow@_K@s@czUZg(<1AvE{ zu`p-lNQrYvZWugwO2V;>F=PVwPzGY|-r-9wk$N~+OJ+P9YJKs4d0 zc9^*Zf^C$LABMgD4+ZDH+m?X{MiKi|M3d?(2d*uV%p)O1S?tOrXw8lJVjtdPH8cJ? z*s)=0PJo}l`X;;L>Y*l#=+DaBLb^`GNMUdD@D?ueqC-&XT> z3u#2Zo8$KKLHqG5MHfVqBiL@QozbgxHJ`)e{2VeWy*nRsA7*nU(y%3SbDRvP$HbZB zFw~WmJ&=T7g#>V#z>xF){5fFJ#L9Hg)n(iCO$mFBwHc90Eg5 zg{Rb2s9W)02Oz2y2PTN~>Y6a%e{1e-nffmeg%VSRa{tH42-9R`v?U&J@BsHZkM6K# ze)UiW$7P7DKFEO&3MFd;n6Z8EJ?K|lFJk-!VT!_Tv5vR@Y7T0t%M0Q;BG`D~6PLb+oA>lg-pJUz49;PaIU;T&Eow_&2WlG zOW6v@J|VNo%|0rN$U(ZR2@XXT#tZ&F&a*!${yNp5pW^}4ap)WD^}KW87^3C{KA7{=Jq^~fX|RrjkMcOgFhIu>PY(EgY0q3o~ms3*{VVXKSWh`%;A zf3COmu;cK%kH3!gD0{{<{}q;jD;>9x^td)NbVa%ku!tz+IpS*)K`xg81`QI z%O|kK*!k_(A(>~&wtA9wwt6El)_o58lexixlK%zTo)Qu8wv2sxDk*Sz{jwiMWWAqO%}ioEhZobNZl2^i%qHo%QEw() z0=x+M=YzYRL(7dVes5Qi^ZSI&K4Nh?T?_zYmJJ!{Cn4R){R4~m7Wzys<;N>wo7!yc zlXBc$#YZ;z=1Oe2G+-fgjJpE&+-#$R)1zQ5XpnKi_>Can$#-c_w;l;CkSn^qKU?_z zc}>?RLT8Ij2VeaYQ$28D*}f#)nJ zsXX0Jp#IST^?fRC$2k*%81oT+#K;9UygF@M&2uN^))$j+n0@#Dvld`ZjQR69sOx|< zGvJ|)x}S{MbI1wfX;bULX19eq>rXNAO5i0kHI8ilBI;KWg7`<_)tW92=9P1(!O}239HS z+3^<&)GK%%0x&QuMjA`G#5F$80Cl}>K5%wMg)Z^r?Q@y?kegJu?D1ls5M&*_PV z5XcuHNz72?upoIIgwLq7Z!eq9pFwc+xl&$;oGOJhdqrk21bhBR;-K0-p{twsai4d( zv2*5)y``tuU_*NR%tnat+3L$GaON@jo@z7%lCqub{=`EFa$8{#=dcaT8hW7=U3$$0 zoAOq#rD#ii4UovA1Ik&u=XMdJSa;^8>sUWtYu`=uT7HACxMhl9f^A>)CYQl)$-8ht z_N9*9HFP5N7N`*MjOpwVBlU%O{pl|!h@T^5I-@QE8e76WisIM{6Q9A+y1y>KX&Iw% z-+|^NUwNv^z43L!7q$aoAGn>>^u^e9m_Csb^zFKlhmisC)gq3XYI_MH0Fi6NU%fh! zN-sTBcBdoLVo+1w&U9&_NwPJ|q|8do{LCv4z-NO1B^M;|P~_JduaT*T6_3oN2Dv{OCs+^Qh zaD+hYbGp43b;)CiafYi5Dj%;#M;WnI=M06ks#KwdK>V+GsRv}u_lrkz+idWg-qiZX zy|U{74$6_d=MkExf=ze-UA;=XPgP@^BHQ5R<18ztIa7zItb3zZhLO_!zfJkTMQLWN zCx`P-HXBBAms|n)|`# zA;QuI)ha~R$Sof}I*T3oJ@5Xk0-Pw|_V?&f(Z%5FO?-lMjFUYX0^E7eo3+8XE=Hn%5ABK7@uVU!~o! zgcs(_h$uX?wsnYvrXtq6D_6~P07c>|iM<5*N3fcM-;n&5>|NbE+9?gH`fKA_j)OQ- z!j24SK7~qHNUr-X!%z3{hPnUCvt#=junxS6kQEbydwkId){y6Jku!w z8}_F*Zsi$2%GRFA-a~c?9AQAN2Pkg`K>e#BOm~1o5G`9r;>m+vB?rYdyN&?GzHCw~ z)VSW^Ka_#*sGL~p-XcX;u7_{gCbT{I`jaoHe>on|e|?`+r0TR!G8lx|d0udwfs@q_ zo>QSsjMqH;!`i^{ggtDz!QcI@oMX6LPp}~(nCd6Gy1aeTF%Qn+h>^Q89jn!iRodU) z%9k|exIZ57jvEG~{~hZ0OHXzZUvVq={}p(RP0D#zt!nprJC1c}Qn9Feq;5r}E)K-2Sya!>^Dq@~Kt7pla zAzu%nK~RQZryCyq^M?!w&^sKJ4FLXk;1DffOae zzbX#@`?qg=t4wy|z@_EG(hn;S@|4Dl|FMGYF|VE!Ou{%Pzk2zDiOeu}eGoguGlCb^ zJ8|5N%IozPSmn{NofWsBEfUW-6e$KV(oY^lA)NN{046@zxtE%m^EVDWPl7+^wZK?d zY;3vCe&kxn;$QlNGnVNE#hv=F0~`(%0(q@p@&f5a2R zMELcw3vq$K{nkGMLEbN(9g=TPoM*zX?oV2$v5yD1*q$Tsk+w5{F*lD9m}F{M%)s_S zdtW)9xh8LbL5DR#6xM8FBO^J_;iudsmsbl>a%{5F#iu>)G|iHISo?#3OLk3MTjIp4 z)t5n5RZ{INhd&?UcvTO4`!OPhu=D~eH0wE(s$hk;1Znn_UNwA`sg(Ftm)0PBnr+ldCf#S=Sx61Ci-9j@9Y}@wp97frV4XLz!hn3hj%?=qVyN{m4YjC-~ z0^~y^LV!Eg0RlSq(&nZVkNrpUZ9Ms^P}hnY+rCboR;}*hiRAf@A+qC#rAvM1Ol|ik zAV+N{iiaXRDT`17N(DNbp*N;VN)g9u&hnP8m)fCx$gA9=5Kq-y9`_e&Hnj)~3+UL1 zO+ZZQo{WYosgu>nz3~?coFy*w72BRWGx(vZ4YiPGFlV{n8s}7fIWTpuNkeiZcfxzI zL{FnfPlo5V3`M32MT(XEZtifAcVbzj=)Fv4hr1iCGJ)&3P$}`Xr^SQZh`~*ej7$(id z)n0pL7iD<6j@T)|Kv4%RE=avGaO$TA0Yv!~NphWlQ!k(0t^3_=^MSb|FLv!-Ry^>w z^g-P@jclLqF+cYBHSGJrT`N@M?j~=fcxum$t1G+D+c81~=Uyd=Y|eSIWn-8_0e1xoKt1A%o5z|m<@>Q(TK^` zH;rS05d9q#8P}`&JDiu|@ULa|eEcj9$)BG@kzRqO`)Wd@H}bplq<8imPs^cT+Rj<7 z7B?{4F{{33&M8q}^WR01gB$boA#%Fa=oaXYH{`vJYR(d;d=79ya95gWxfam^{KL0_ zJsbacWK<5QWf1XY=;#MCQNBo3HO72%TU)bgn;GVQ$JhC}yfO@(fayZ0HMsMio4&BagEA}p} zEZ%5XkIHuQj?t1h{?k%i$DSssEx4k_u#N_-rEa79WI}-*+-vwpomaFYYr&`NpFjd) zd}D*y!N_qgPqsm{dX6QvQRVwc-uvwPmNo5PJbZR&Odyd_cdqV(IDKiu1cmGK#k#4^ z!w0n2EvQ6jq>){^bmcaj*oTLz4nmht)7s(9)dqnA2y=Z0;|>YD)x&-_U?s25o0}$l z87vkIpYV}XR}kchKF@=-%cEV9wSaD!d)}V|arO)$=Q~Utlc10Wx05Sd@41MbY{KGa zJNjA^6hGd3Hu8hwnPH_b5mmhP_NQI`Ugs$m1>lXbDRyih?{YR8*r>Tbm=vzMoUZ_T zCzbzwITt;uSI{`-Vg?DIz7mGH&HniJ(gLr;8H*NSx~LzW*+8Oe7I~R&@EX+f-K*;*?FZshG2R zi~S(324sJ_JDtPMoYjo;$ZDJ%jC`Wx=AxCSLM}$1+I(uu(BzgrLr=2q1T#lqen)2G zyKPPxJF;u)HKU)wg_1mkcEjT4Vw_q3h%4s^h16E>bCb^qZ@#uQcAtP>gRu-g>Ng-Q zLvBC<$KLPor$9| zK?Q;?i&Er*lz*B96_Lc~^$}W!yhN=a{+QC+~#e$P4_ z2r0xe(=WUtN_!PPI|>+ON$&fn6y=lEm7Umw*~$)8%~Oj+#7Yp}j@V79I!(7S9uuj5 z?g4-J4F2XCrH|HfP5`@vE^%P=c4DI$NO)IMwsqS#NAjU({0FJc=I$@g3?0AlU>n7- zBUu=(V?f2A7Ua~OhaRZhW$t}eO`L^0#mA^E5_M!(s^5urDcaLfZg~kYL;H7!udZSF zFkuBB)r!T=44TRt70;Os?Xs_NbfOihBjJG`+v^0~b1>bD`)qB{E8BukZAapr5+)ej`)QQbu=E z{Mn&h*3$Zl_JOLOj$VE7BT49K!HMCk1l-+SVN=D-+T&$bwV6}7v@6R4rs8i3j)@6H z2&{&~;j7V2x|K&%M#>~pT{IAeS;NmS#(OrS@k%x{H3IDd(Z;zUU7g^~FwKKc>P~!) z_L;4r=M}GHdiG(EeT+9ouv>Jsqai-j9}%z_s~s)L3|^=tJ^|EYc3-{vjgpeIVo!E# za~uQCOU>nUIvLf+BrT$l-?H=U<98xQqLDsLpVDq-9dY<)UaT?mV+A5J#D35vh?o&V z*McfdN@;(s@x~XFEY5W^HJD!GPjP8(I4^oM9hs=)J5o6BuYrCzKvn8=UVcLsnWU?P z3MX}FPn;Ucdk{mpBKe{aP^z&twCh}7Oc}|i<8<~YA>BBXa5mT;JCc@1-O)=AqPWj~ zlmDva^ZM9v3h55;Z^O4V=0q?Hm>w+E0LyZ7zOmg}p+;c)!G1Ltrc%;VR4+g>a7?GtBaWE3rd~FSILM*lbMnVqPvdyyX>Z6)hp(r9Zdvl{*FWsXckPJ+8!WD9eQRA2 z*vc~f!a~y11S41p^P}gKN&zyppamwG1!daYE(9=+O=|c6R4FUTLW$}84u0_nAq4ZxI z5P!KOI4J)SDg^a#e%v1fmyGg=*4-w5wvW*C$y3 z&!WOrGcm^GlS#S0Io$zMcL(2PIt%(rHmHbzi5K`&qyn;ob0rNnASio7$z-}xQpY4+ zBtjO0J?*|ZqCt~?YVSe5@UC;t`tqBdT@QU0r+50|`W8>I3aiKVf`4Um(i&PvEq%r& zMM$Cz;i?=$z*!CkW)3Q;S}hdT2~uutuljCMieJ3>VFGc#Z>)|%W)<0liK^6;5ZIxy ztP3@0l=q+jJJf$)Recbr8{}#A;>Dm!+ugWe>df0WR2c{En4Ix)6P+rd)wFF|J5mBv z2mD|jOFFD>4+5o5G-1*1;@~D{nCjh*%vS|e@NU#?D_48KVs94V@ILY9FUy_qSQ=q# zf`@p+j*#p1LJf*lho*F{TsRcdemGYtXhG)yDWASydsgqzv<3;SQwwT(dZmq5GN<14 z)hc7r=bc$n8|W#|-VaC_@3}A?b#Z5c5paFE#9|?O{+>AnADJ@JL9*JBkppK=)gd!b zw6wf*%XYV8d^RgTZ3`Mx=(AG(S^F_q3Jx}{)$TuE@tIlNac7@fu(DT1_|M>K+T|Ch zqB7(9H*Gd0Y^g?j2+e-Bt`qRaR-{x5yp6eI{gN>AE{43vy!>NuE1TgwG}%k?^@%&oje&p1=66jAkwG=UAiDY`AeU8yP zbC8e(m0ZwOm(hOLuw7@bMVzrZ4CB?-rDHfwqY7EXhsX33x^b)1)e^2dQMnXXQJ;o%jU@F&3U8~`o`Ov!*-O;v zWs$u5)4dl#2^wus${UiN5kir3rU8iFQD4U`|ERo}q9b)iS|?QXt9(ttL5FkI;sWF) zOR|UMPD<&s>wVRRbIwgCtLF}qc;`G3ZJknrS=V+Y4CpXQxn2wU`}{6S1}_FaJ|{x(7|e`lWq7z{zX1kzFl^(n`)Bf3Dm!j~3ZrN>*jYe4#`>@v5Ov z%*3J1lP_HRR+^fN?#J(xKZ_0bd!Fq5HG`a3q?OZMbou5DpYPp5A7LrdPHh-Qca6}U zH`#j#(Ws|*GY3>ohkSVgydNr3#UN1}ljcflhJuGjGT$Z7JWJvCrqJm#--c4g^GI}& zDb(OB>m|sK;@hvwpNe~t;SCkH!O+R^JrQ?DL6GNfEe1j+7f7^t%%(U}p#h`Z)%6q4I! zXR&lVXR1(iqrpDou&yDG15Ng)+?tLz@!~;PYRmshw(K5yidu*9V0{!gnD~qsOmsNY zCH%2%!q?*#LHNV>;UgnmqOOUG)y#;xu9}{qYI_D=N58aXr^}6@LJt9osnXta;d7qc z!ylwNvv#xJ1YRu-yn4qA+x}sqo=4L(J~T`)W9Vj)gZ&v=ndD(VnVvFwPwz+`q9(Te z#v#xAFUXmeh(n&5fxA9WbOy1$+2mg&<;EGDUbMez68+BU;alFr2mcv$iAKIFJa~!H z({;G8x^STnlcqhm4X2>KOyP}9qG#yUsWTttZX=JLEp62LXFksK*`2Q_0`6k*%G~kj z?~`{5WQe-Mp>7x6Ud1{3tkmjQwph8EhYKGmOv%lc9(^J z^Xx4X8Ls{`mO1qE6^)ANP@}U`miW`Y??rOg

K)#Ix1p_ zF4B0=U5&J4?UisaOZVH4hS|`QtfHpfNX1DJ`d&%xiknxtEIs=)yeA~>B2t$Vyh9yY zJ7Ti33ByMW`ph4|Mf()W9H#3~4ZN9ArHVE7Rr(vdb`~l6I9kO87w1d}pHPoj5;Lfz zY%jkeJ?0l3vbYoUwpJ_n6=D$k>Y#x%L8|@2U42k1h2$R^)aaaIy7;u7=Lr)?U_7rK z9y#bbC^8Pj^+}#T;?9eU$zj!jAQ31`FsyZEgqeK5G@a)_0G{dLnZ+E*P9%@j?g#(@V`A_u?>51((H8(*T&+e4+p|DWd9o7-blQR}4{w;~gdvV{Sqxd}29gr<*`~Gz=c4{OhiY#WvOidUs@Cu-7tEAa{9z9ZUM3|7 z**MYFPMTS+VWHB%|%TP!n+KfP(ZAYC;G}?wkOM z?|s+0>)!iYzd!E%p5+oa=ggd$J$v@-*?WKXj0~zJVbfe<(mn9!?e^swA0N0!x@llm zl$Y{r*f7`dRf3TO(W1rOR(%ufANe43<-C<1fndqeA1f-L=UQVDH+G;}uH|v&^ChYG z*)w+WvKB;{`u5|~HSjbmez@ieUk`Gs0e_5tx$)+X3dloB?Kjg4!ulmPJrl$i#9AgGqqTEa!gl%MOzCYxPo_*TZDgwh; zrG5VZ!~OKw>Sk&yN({i%T^Y8f7%Lk#i&=But9Srsp*_zKN;58jVqJkS*XSd7}-_ss?w`b{u=netI#j0zKamE zJbf=n=wV0p?Zdh^kj-Z~3Qv_FYXX-aBt>E87Q2VO9G~2e;;utG;^9ZR2JqASzyR<+ zwR>B)*Z*_z67|T*+5Bm^M|@1*SvJ2d`0soj&PNB51g#^3w&0uzRu6m@%?*Nm|1g04PP)$Pr~}lso`)hq5g+@@gOV0yywJ{ERc`d314PQHLfcd2*noJ! z;Q)7hXSd^RiMm@3Oqp#&=d?WpY5BHs+h&n~=0k72fpc7bR&*C(7}eWZ`tsPx3kM75 z?NGM+Pe0cug71kh%-+~Ph;uXx*dhJm-IikmO|~bG?YLeb%%3rFk%u?^)IdN) z@$^Uk`S35kBm4K|ycMbbwN;~7cck|pS*B@eGek#p6GiM{C(4&yr*YX4-xj_`>guzaf4KW2Jj*JPpOLopNabCU$_d}9RaiPsz7kIQw~zo<^V_n# z=zo6D$<+Jz@ zwJ*AdIP72Q?dJ;7TLCKCeS((-sytp>(9!V98;i=zTn0~Oqldn5-c$AWtx8vNi$7Iz zfoNjzd09O7M^1l#jgO$+M1my6q^6<#yHh{W0awIHa!yd!F4k4Pdp2`05J`!CP9#s4 z)eY<=r~++MImP$<^>>O^npb-DQ%583eY73cNwT_Ln71@#6X|x$c)HN`(RC{Y+t*00kO0Iu0aVtVwsiA=b(2@1&1m9C0Az!E0XRM zu&l;b*nKjzHQ?Z5(z4yV09SKH9IokhwAiPJTUka?AB)8&KE`F^$ApUT!KxRAN2{*} z-I(#MIY;gQme7~h>d$Jp4YBgn?uM%iX$Uz%2{!6wu(aZU6ab zztVt8AYSVz*jt4=hbNy?e%f<%$T-!ir;y~KStO$I)3ED_ZR>c!9%H@*p&tP|*vDmb zV8E!4KxTNm<;FrwlXOs!&hWxkwrvor!*iaYnM^94h0v8CxON5P^a0gnXi4=kAu!BGJa(8tMw^ zqn6rIhj4{iQvpkzFI@s1BJ5))I*Xd((gNWAmkTv2ojQ4Jgcp8oRmtsitFmhRx%r&t znx^D!H-fQyodR5crnY&@`QaXCHwjA#X=mR+d31<%Gu65qOyJC_aBN~Q8qmvXmE&X&~62dna)u{~dj^x&hV z6eTC`soQ=g!0tFND&$-plZxNr!Q;Kvmd$$=YeSvkXZ_$~=+SNPdTng%G7O5p`_*8jjbmoV5pzOziXay!+}0ZHKl@+?J=3yBA|fKkYfs1saad zSDQtX%eh#&SX(}x5ba1B64UqSh#q0@3C>!gUDFO4;$^U!Mk_*((_iZS(Oh&h+QJA-fW@jjSRXeH_{P3jS3+ za^TF}N>A2Vx$pVYC9@AId*=_q#oCF?4ws}yN^5rqVb|kUQjY;Yc9O$)Z1-Dha&^oj z(zKZcf!;^r8#m~7txFFb`{ol!;@bX7;XqsT)?>_DfF3>cs%!KQWcK(82LcgoX!ISrN)jIzcg#srAA$B&IH@B}_zR`~ALdBfmb_=NLA5P2lv12!-($G|OPKMMO-4Inff21_%BW%tKyXPNSn-0IdruyKPYem}eSIXkkgC-1br zg=Ph5V97vg9^Q5d!!{0_crh-0REa^d`=i{B1MJn3yj=;N`B@*|y-XfbW;9l@>#}lX z$NrF^=6IB7Ul-c3Y+xb99ivXk_%JFyFz%Z_a-|V}aMKGd>*=%S&xbNGA{#CP-NRZ< zO%3M+vaNXzS+>1^-Ojj|t*pnlM1dj|ugH;3WZWY@G!Of6<}gGo$6p$YerUM6MZss! zl;a4-t_Ia|Mn~hG{T)xy8OY=!h27cRzU2aHJR9Boz zKk9pvhO+K{G1WIWwxc|x5{hT?#T;^B&Q4K2Gyd$93nzVyS0dDnC*1P~ zV{DsrJ7dINMIhphF>(`z2K$Pv_7rVuNKBYF{DrJpd~DCwh^8Q9l35p^t3F{xT>`?~|LR&uc-Mae;wog?_uP3hJRC zS<;IA9qYym_SQZ{sJ?|oLzdtr-g_U%Dv93ps7wQoxeo+YSSeMk4LE762FkJQVm~I% zA2{bRou`%pOgnRz&zRcZAvMM%!W$3OaGc-g(o4=7YQu;8T=npz<$s8u?=~=5c>yEE z*`yiB3pP$aLp!IAD$Du%{yX8%EA>tDdf7?G39h0|!et2{;R?g4^4=w6vKohH^(6#jjd+sFDwl$!iMkkS4TmP%CFrS^>F zb%+(o%7*&g2R^zY8F=h4dKLrR=vPkYf;F~j;&V;jafa88^`&hR^!KpPrnTB#M2k8Xi?l zjfGqBMlA6rcs7kx*nt>HodE+o4V;#!hglD2u=~%ZY~^~D<9l&_&+;U9ROi|RL00st zxK2VTHo{L&$e$VMPXy{s4JOe4 z%S&(2(mdRz(FFy!{sz+2%++QqWqOq^)W*Vt8 z(z>d~o13p`dcY0-;ZtM-@l;TFXuK<>NUEtEY(It1?ci0!bvQl~LO#cRsD zIZe6e+CXy9wNA04@whA>lHUNn81Cxr0sIKw$NRkH2H+(h_K27PzuMi>m9Q{_27bfI zqJI8Uqy$!lSUcd3=!6eu_$Op*>~rCZ^V`wmn{BUB*EFB};fq6Mo>v2o_z}-%n<+7F z+kjrQun6y8%yNp^a*U=h*g*QG%iAN-pyNB}b>lRaEL8OE$i6ZZTt8<&p7S3T;PihL z(vP@f)xe4UIV;;MSt#g%orp%zq)&0FjK~gBi(XL%bbhi+#%Up7`@iq@kB>Teyo4Aj zmMl3yb~K1hJed1oPky7L1d40K#^iF)fIiB8yu$A8H^MZ|8{irlrh4IsM<6aT;y4gE zN6tA?eIHERxR!QX(06m*jbinSk_wGW6^}4%=T-PW9ouV<1fC{L_$WvCZu*>58(3X$ zOzrx?G_O>4+j{)Xg=3wE77&N)5`!a3k0_-Up{bk{;hhVSyY`7)%WO+&&am)jJML95 z_8Iq}rvYxSzWK?1C-<4mpd@e3D&IvEQq7eIhF$!>4Vi~V$=sya|6yl z(ehRQa)eZw48Y48T}DPa;zb(a)-p*RmtV6tR??_DyZB@MOkVj~jq~oKd{Fr`7`!_~ z`D!m`oG$llNqe=2#7HBhsyP0_$@FS8z$z5))|uw#D?25ipMBxA7l;~SeyGj#{?!GI z!*yAM{>K9#3;4;p3w4)6mOPly%0M&)`0hJ48!!6v=o=1x4#&ZNSdw2wNiKxRl@0og zWx&X;R$Ls{NFWR+xKCpK(5_oY54~;XzIj5P+eqFQUZ>%(GeaoPZ%b>Cf!n8O{-hh} zWxzCBkSQQ4h5TyN{JAKeO$fwIR5Z>W%gI zgJp@AY$*2#{jyFnq%ZdS=1h8y(U3?Lvv*6q{2yeAS}RdnJbJTe6^f;r?GM@xfD&Nc zCFC(ONGcR9_wOGO?DRia;wF0DI5xsL9=2<5mGwvr(&_8BIGYk-9!lpviy&5hTF~vu zHoNO1R(eGwq88S~FUWo6*}bhn9QGYmskze;7vc{WxlayqHToqFI3{Em8H`}~tL1=8 zcCqiuk)}yq`o6JjS;80wn?lM>y`bK^P1pvcX!f75jk_$rx+M9&&(yEgE*CKZ0g0Re zTiB0E1uv^qzdz9g#|B%{iEJQ^nIhJP{02T4}xK5EHNkhi*hD#s9d>H7Ak+m)zfX zAQJ7hD?fW_SHh@;CMw5@&J`|FMIt3i)Eo%jgHO8Q5X-Jom3)Tdrau2E*{gdOuSigg zwVG-)|mku&a3inL~SXt}$&_UgyisN(F~U=NdXaGEwjIquBVb!zX8>l`=n zRuAP!_V`g4pZcaGKXNqy7(p}Cw^bed6=!9WfND?O3fC<^l!{8tFfybX`wyG zSpmq+vx=Q^JBs*jW0fzEUE4;Rk*Ig8(2A1b$ZU9|J&$-+)Mta?gtY&!#VqFYy~PJP*nF-b6Y~Cu8ZSa#6eJ@^ds*1=W;s|^^WVM z$el5oEb?WZIC&`T^qUaEI~P?0Qhtn_Z#!U4HS7XsS$iyMg1 z{Bru_z~K|}hSu1hnE5)15osU4n6EX)Rzw?xTr5^{pZ&e2WHurB`R}oXRc@>tUYma< z^fo7#?K;YZkR!m(AKh1_V5X#9K4;pIuI1-O9jyuahQ_$k!%r|Vb@z}fn;zn6p)e0E zV`nM%slbKW#C(%B@{Jh)CklB8VilFwL&{L%!aF~8W+R{-+$D6mN)8kzAiaYSd^Ev? zzy7H++`|{HYyI_1Q)5lss7Qcus^df_aqmUPo96fMa9Q~DWK{r%l>Z^F>s;kn$DU&K zngj4YTQ$PRfzpFD;sI3mmlw)bq%(E%U6Tf`ngjqx4EdkFt(>>oY@90}vhd~oJ9)XF z_pV9Nz>0mP8nDPLJ*&>^D7FGwtBzq}n~n^JJHuVNFbFuo=A7LQrAx3yF-2`g#J|V@ zD~SB{UkckKfZ4=UJ(xsVah|J7Z7rfNmoIqv@Qva7Kp^dc9@gc?Vc`At+*8MkGn`L$ z0CABE8j!efhkRZgf7wqt% z8a;O`G`>)E(P$?DgRg6hRlJ7JE3M*!^%je(p zAbtBUsQCMU1id~WEIt&57&`otT~V>Q4aG3>oS84~$RH6!`b*~h9B>WpdIZFKTZ~z> z%Y7R>R1O47-{hyuipEE?h>EQtPn2@*ZLwB*Eh#5Q9dL(UX+BT?ImnenA@ua=yydE= z9GKL~(xZ2SZ6SJQwNcGHK%qSX1;*@u`i@u!75eo6is1x~_Q>!?mzIwN5dYhdX-{9n z+J`8sUU6&Ah}SWPhJ^W_6C z?$T>UkQk~t&zkP4tDz|S7`!3Pr!=^1&|4B z{`h#dFU~TS8llkrhEc`pe>$5%I6nIdxwl4e)safr zAG65wkk`n3Nq0pgTN_07{&`9KUF6Qib8_`zW>Es^}1QsL>sr?(YVST4~)$YADU*vanAdX%ez#25U7%&0hwTP(o$BOW) zY|aL}#l7ob3kk8eUgvl=!bH$u7!FKrl7=8_iQ791G&Vq%Mhk1VLU>kyo?NdH`nfMb zun&Z^`~pcIkFs(^>}cHZ5S?1fBIZvVu!(o$d7lC8(2m~N0R9+F;b(Ds{P^)blH4mp zPkoSovxZ~WotP#dAY5M`D-iJsArJOj{Q&Eky(4g8lq zS8a7uWn4vrcl0+rG|JPO|C~|JguVrVGJ)wk!TCQDzX;k@3_WFL&>{6=+cRAd-T(VK zskeW+W5`2joY^iHaDB?*q^ha^g6RyOba;4*!Xy&&!B-}CDe0RPS zPUvg%LygvIpL03|V;Mbb`c;uL-Re!i#O4hE4*+KYzSX1L9oB9t^}pKzxPF`TDu8*z zy_^RL>NY40F*5&RSceG9=wWC^16*T6;f9x*cLT)EFUYO_%bS@&+r*3Y&$<|sZtXR6 z>gZPpoDE@mxQC8rxfxvAh{ckErbiO6>&7y%{i)c<4kV8{6^%za+N;_YtW_q5nK@TX#?2xYs9r>j^suFOZN8&P)TNPiMQ zmoTt6&HE$XZOo3s}f^VLh`*e&@x}*m%Rgj6Q%il-^3ZP%V z07CAw+-`kOeTcvA0F)8rJ6*qw<{tHJ{slq1k`q|ttT!Uxz@**>`}G@;^Xf5I`2cF; zv^JX4;x4%=TD%6<>Pi#p+_&-Q5{KK;QqUx30npGF{)Kf_4XntIrbH?hBfUYkLO%EF@=Lhg03djWDp$ZRCE%K*-CS=F}fB z(P6w1kaaTZ$Qm|A>BqhO#2p*Q_GEap%~3cI+rnCZC6E$-m;O!fD{ctjtf1+iyuG=h za-II*XY+25LEzW34NM#lBy7ynOFuRaVoAC;VkeogsOL4K4DyI|@K zkOAsSw@5qtyX+ZV2&t5~kDv-X$4r00i03q|`|eED3jFT9s+(hdUC-B0Av3(kv@WDU zbuW{%0ZN)Ml|-rp+9?wN9m|scv%-39gX{7DakdJYY$#Kg>Us8F1etN~oBSBYf|UR0 z{w4vY3Pk|fNX1L!YIezs>4vVtv;m^J^b!PWK0IqJ1;Mnhfk61!@yH?|H5m|3xl4L9 zB7Wq=qsQZ13Xf*rDJ-_Psk$*UEpP~w#Q|iyZauLNae9i<>b@89=|I5x-+1pSHmJih zm8JE1EsiMjfjd;zW>aW9LYdElYEx2(x5QwW{!CNbJ)~bDYRRc_WEWR!SD8&rIW_rB za^W7ddHk4J@Nsp&5MsYgf8{TT00kh_kQdaehycL>Yc~L}yi#e6v$y%{axH(vA}UAsIWOmO916LIQYuB2zX8z$pJ(altha` z@O>b|W{C4s%PG8A^{KlLp_VKV>vmd>)aw8kqpQ8$aBK)@YTq_1Gbr8-Lfe7>&g;G0 ztThQ50&qiA4dfy|=dx)P=U(E8+TNM2<&2`cpykZlyDP zimD6&!a3dnl9l~=2JrH-GY60X(NM;{)!8G`Da2y^H^^1HL>-)cv*I1Mwx7xW$E71_@f$?*jLfU8sF|2DyLeq!^RIN1)g6Kl=!p4-3L7xH}L{U>=GaRhJz= zb%7CYV~Y(}!PE>QrRNLXwV~GycLiZbJ2xna6?@Z$;jo#|k|4k6nGYYp0>n&@+@vdB zx}iozDKMd**K(lr*qW6fG}xaj;s%i4j4{`Uee@}F;}VYNkp98MNU)GB*cT9Rbb~7L zUhk{R4_gDn3u^=T8FP9Qm_6&4BK$2kx_!u(*&!Grwqw4?xdX%vLm{r~KLGh_80(oJ zi@8!o3hM^_2f{^TQ&Ur6Wg=irVBnr1p#PuC;Yv$;a<4Zy3W@>HxBM$@06MRnTM9D_N+Tl@?)rsP@G-D`Z@#{ z*6lgUFpdCr`}<;O#Q_N*eNH)V`c@Ang96p&AX`}lgEoy?dCtSdxfO-`l0R@tZN+H9GQz2i5s-uieDhX3Isx5D}HQqAzGK~ z@6BZfY!*tZ%*K$Y7E$`b8@>%R^&$7`v$303(-(lfV*Ws3#fH77Q67*7^|+`=43h)E z)*rpC^!}vz28N5UoWnltg;e8&3En?FRxG;8>umd>bWNr9{gD~r7{9%Ji{+nJKxzwc zZXd|*wK}Iof3#iu3tIp6n_fCNo6u=B3xw*&h`*9DiI;o@bDps?i~leYyK%}JS zTPl-JNgZJ%uh9h~Acu#4rY}kv2rk?$a@PY41}2ux9YF)A)VCjSVUJ%Cq#9aq6R84rH#mIC0s9 z=)G#Kg6hzL)_B#PLWKwj9JK&xdIBEAyEf!>0Im~)<(|Y?<1JbzI?GQ&7Cd0fdoKv- z53XmpE<%#vGGhsy+^O_W+@HC}skVR8eLfq3s6sd<6pKv<)H4i>0CJ^fBk$k^?Gbc_ z{kMr~DajWbulZ%t3YA5nz3W=PjA%+D7zDNyqn(E9B$dg*}$~eBk{DK?qGRKj6Sr!fL8b>j9P=(#g8PA7V4|R48R3husLfH_R_}U z_|%}FBXf5Isol($O_?GkpmNbWy2t21&kbWM-H8Z-R0960p_nXDCW1 zq|feGRNEcn z7Sl?1ozBmn26?YeQR^somh0&f$$UtKbuljf9O=9iTulj77#=NvT?sbyr3{<_S-3JW z$O$3~NG0*w2Mo)tBAc6IP4YPcKBbc)1ov!GQ8Y`oGka}|VN=r$rgv$y5YD3ddTVMv zrd1Bci}fHU=(UwCf0u17%X1&h{a^R*P=Pz-6EMV;rAw80xTAUuxgp^2a@4!q9ZkrU z^aVQJ)#0!i<;BoEPtX+ifBo0wX4y|)u}Nvudnh{8i0~(S>H}r?L9(d2P?*x0{#-g` zK{*H7psrp<7wHR4K{(NV=jNv$vY^nR?ja!}0$K;jGLw~ETiq_T@V?Fm3#!CW0;bq- zMWk$~|6-gWZG5Cv3e_hoM>GFj)fO!G+rCNnV&^ zj#Wc>xZgtcoQq*|ubF=o7yeD8RmfVXkJ;`cZr(9e6cTa+LSh-Td~FQ&k(?t*Tb9~^ zc}UJxLFttvn=S70Nq`Wtl+;uU=FZ1bp9fqzrm_%t$sToh>B?cp39*L_f=JFGlDfOc zqAR|z!wT6(v2s=OD#Wft%uS&e7b%t*B<(nDI5;7`15=YS5*pV-Kzz4RNR1@5I=NGc zH}Ui7K@&xMO;Pe~DYHf^-1MQEw>G$}rX>{8#&O~U!f?^H8arzP9(?JOq`ji2 z)EuOLn3Y>FSxN0CX?v7|M^aXavvJ$xQOg$w`R&?rM}tb`XcLC1ZZH>j>NE?K`sS#`5OUz_C!;Mdsl2l0G1Se& zw&7BzSc)B+tYjk0DTl3idk98pQ|N6xSG_N>_wTBc$pe&MsN=5 z9i8({xVXDr7N}unS_;AJNak)=-jrdqZ^t2eFyU)C`&k%B;NRL`4#C6$rC-Ajikqj(@F> zWJ~$V!tWKU+2KIXVY%YX@HKI|}XzYSFS+;6=PtqjM_G z_;Ba81(>Q3QXf&GtN~B0G9pCtmA;n{Ur5n1P-cKPp<53r6JcBv8S;(b?nGfpAmB{) zZdyvupHhHTu~JZ?Y53?GphcgJaabWFQHecLi}TQ8Z-I5Qm~!{QxR}T+ z;b018C4cJbU;QlLN-P<3h+a*13#6mndkMKzYi7v&lp}nOr@;HtC%C~LT~hDAY>MNt zZG?*V-!@$_*kcH1!Vx~EleCIZ)&?qsj7PdLl08~|FVNKHVZ|crr~t2qlquc^&E?sB z+9dfXqAAv~2#=x~mVR4~>g=%5zuT|rsPHml%)LX-x_P1qk34mV?l|$-6$ZKpGX-RD z1Vlj|g#I!}orPh+$vAYJ6istb@-W>Cv=xcB(BjkXm86ZFg=XnNaK|yG8c8iYM5n^~ zm(*VMn6I!ugz7!=KCrC&|_gx~S-o!Gec}&>2dPFhDf>5x<=T{v|BMXzQUD}p=-0ppGlN_R4JA~}0 zDRQA+Vi|xqBxqIvj;}@@F5W#v*}2XZ^hw8dj0b*sd^!W6`fe6RUumB)te*gK3>s-v zHlPA?QM?=M6C~OgvIrB1#-a3Jcz4I9Qxdd^xVXc87B1Aa`Z^rB>5qO$z9k3=kId3L zdZv{XXf-u=LZJ1r8?0GlKzV2@rPd|}CG$1u7g{65L5k4QpVAYpE!ahHyq1CZ#u^Eb z$osENxKtk{BYY1_U~Kw@Z?n#-!J@oAd$7b)J7vET3tjrsCSuhBN_Kk@>Bd&|f+z)8 zPoX-&_McAK(f2lQ4$>3P&{)Ly9QxhLh{v;Wpl@6QA0h53KlH!mQ4@Scft;V(8g8?k zQT?p!U&btRUa?7~8W!3Nnn0h52d$W;C^(bfQ-%jW3fN)*39)-n5F^ijTjKk6p+|kv z-gD^T=}p2{I956gB6)kRZAln1Di9UtIibTN;3-y!8IyaVDtIykdUSf`ft@TtTf#V!mNQNkvYAx`e+}!Gy@<^` zJ#r~@k-Lg6e!h!cC1ot@Lm%O~yZ*8UJu#lvfixJ0QG@KH!T-2q+Vv~c3_7|UyUaIa z2mVLhrUt!O)}}jtKrSkL9ZEwk!orvNb}yo66L;t#dUTG$9=n$6bEl)kR;s2ib|8`C zSl&g2!!+uD7I~%7u1aGyDrxm&_Y58bN3*0EN7z@B3yUxCwkOjBdK2`($;D9P;GkirxKj$eU(LYHT7M2tIz`nmX>5!U zHDPoIq$OW{5m*)Oa12vgU_3XGG@CdfH{`rt45{(r{7Y!dAU>AzCkx$6b%)MUqd{~l zsYt&-!xQ?Jj^y9Rbu+y`@%K^jOwAx4{rZ;}Kh42O-GALE04ECnb>lwBJO9@W=!k)h zH$20@!Dbt8DBk{`X8fVY|4`!(Etr`8hZ_HXqQ>!!oqf;qzd;z?XlDDrBjG%QA?Xdt zR?iJ`Py~|JPnH%I7~wEE(HrEOxSfEZ5Zi3l#oG)qsbjc7)EzA0!#E^8S&}B`Gs4!w z_HAg&wdq=MQUb-WjJR)8`8yasY<*?SiQf3q=&w4dRzIAf8mi3h zBJ~$4D3w`^cbbj+IO~iG+z|S97&!F^{O~NQd?ldZI(CC}?QK_)^)0fPO_|2ni~Wpd z_W_g1y<`|>60@1%{jpgkY1qf@qg5e^$I9TK#SpETJcmWU7mGqfk(qChV64zK;N%$a zBrJJ5L5}`8z5qL)+{O#NTk+(cD(Xg0f6<97panpTSE0`!Lq*;`9#ZVmr@nKt6jz2l z=Y0u0+U43Zg&UWw5+jBuLB;)7yIFtDp|^)L$%xSEQQXu5DL#=bHAoO`GLLN+{m9GptXs6QPXQvr&d zG~BX8Tj;;o;m&j1`i3rj`XioOV<6>is9Dw%iJjnu)$7%2YxZS5_O$utGLrI*%WATG z-4?}Ek+#ah`zF9}Y}>Z9)^R20-};5;y|p(i`t!vWcF!2(*cx;Lwa;Pro!KAJVeZA? zQt@E6GUp3Z$e-$Clqj+8@4MH6Awg3VqZoQo#g=Z$Bx^DO9E-Q!!W?x5hkE+u|7LFR zkr<{wIu9#2s~5Z-aY4at_Q}r~Fg>>X@lfihua3=CX`r~Zinx|jG1UT&NJ#8njBWR%z0U(N9g989%d?R%?LL6(sF`jY&nKbu$1XAHTSlO}xi zwHjmh%P6rfkA-EdwvjhV@*8W_v|s!C^^>dX$jN( z?Yf&}82T4K*FTu;6QNsY`G4Vl$!%y@H#k-hyf1vn>zSM5hANv2KS0ZU&egJsn2gp2 zl+}b&CwJI}x>;AHxz3$ye-XeaepM#lZWRBU0B=?aaQys^jV;e&yEp4i@Q&YGV#Jfq%%2L+_ z(|N)B_dMh=^J3gZ=5qdFdXiX=WffS##%u*4@6fr$|-u%DKL`fHv zd2imL)mf~}wyBIfQu<5np_OrX@3^&ns7%8!3RsX@W;ELAKqVbhC|lrbdlW;L%+%BS z9wVj@m$#Ue$NNQ{K8`haUF=7${M#pVxX98yssY6YY)Anq+H`6S-!LY@w!X8dZe{#` z!;2tagg!)iohM}0A6okj+Ii1;gbeLhftg7=cINiGZTpsJ9*ypiZdjIlXV$`JHHUmM zO0w@N`x=IOfae-O293v(90TpU5L&~U#Cbu|?KbpX`p_VWThg5O5|!PsPY^Pdnacxn zTKKCBT92M;%YtK=rWp_A+jU@C%d_o3BI-{?mN(jsXPRKikVQ>0d)dZDTB!}Es~Y1t zDa8jUQ(vt`pAtzmUzVx=kRM1LP6Sl$DjvsTHnbF&=ft?RsYON631bt&sf{wksGwz7 z*&am9cuy!*El&~5ff0TRda|)J^6y_bxG1duv?89Cbd(ACfM%KN2M(_|6&*T zlC7rKRv;Z`d$>w6jXvPA1+;=o0*qPgIF`NudXw@0;%3M_x>608Zr32;;n}&Lnyx!y zJatrwt33aw)(fpehE5!4()n`L7?}=|u}25%C-Niz7up86yxH{h`Rk#1CM~jtC$V%_ zk}m2JQpY?`j!Y&-X@`>L=B;Ju36C-wk40HGpGlMOXtA9tiZva%)L~z|1GL?szB=O; zH3JFylJOGI5DQG1pev+u6pbP#fDlWhwMEZv6VY6Ig~5Kj<5=?2IC8oysu646ojzZi zqDi;MnKtK~{<2ygYgxPulV9w&F_>3iUQ?gN?M%7yGFHZ=M+Yohy!%cgp9oS0FLy1F z7x@-g!Y^@A>Z|Kfx4R^@$zF;bF4{33&GPAU*I!)XszIdkEYyUbp1>2>Wyw}I-3v=C zJ+wMgE%D2{EzS~7*VpZmNeu|J>afo=H#Dt3Tu?rab&diJvB`^m2^nTnL65Ls?fi$e z3NtTFWY7YNbiq)atgnMO3+@W_TsZarNzvsYLweUaYU5Y}2)*#sfObmn4|Jj4DPgUN zrGh<#&5t}hOu5Dp$VP0nHnF{I#VtUzohwVTgB7pTbHA`R&nio{DUXUPrOvvy(CTb7 z@m73ElR->gDSQP%a})Ze?@Zg+mYLNF(&vD)Iemq2H+0a@v426fNy4kIDM*{>6Ou4B zQ6!=?5u>1O605Wr4r2$s2}lug6~i`ah3QIWt&L`sA?Wdn4AfyWDj<#6eZuL9B-%)M zIBeu}y>QgM)#0V6K(fk%2Qq8t(*PTAWx+!o@6hf0>rwWhreP`>s|;eQ;IksFUod%^ z0#>(lNFEzKnB#grk0KLTz9o(ZF`07F14O@vQ-3BjqAIjRMM!^>WTN^rxyC8kQ zGq{$1{~h%RirOcHzFWnsZoYDftBA#55;a%j>A{s-ANKB*#OQz%Hh@m;%o^o) zZgIU!Hc{!~JDxsWnx~-rHcPXGV%no$Snegdz#a)BT%7+h-xhmsKFES>H7!LGr(iA1 zsf}%?>!+kLze1x0`rzLu*7Fc*-77FQP;X5qvY=nom~nstPY2WJpC;bZMy8`yauSBp zK@nNY<^m-!xZs^QY#&1`oqIxUM;^WqxXf3^2Ig&f9lOx|Y)}>E%Qh9g0@ZP>E0v3< z$Y ziPlI%TzvI4EodsnhhA(?qiP)v{SMD%zi#@rMzXQw0_mD*dkyiJg{T^~;IZc6Pz}DG z{{E$?M|TZf20I+;JGIkesjca(G8BUUxzmaAg^4th<=dK0 z!s#!wrMK)ov>!G8_3JmXf$4?SyMOLCPKv{ZuPbpz zxzbK^p$iBfdjGV=bMioJcqTXFVB|yyZECb7Pt|QUbn6nWBN+cjJr#Cs2_|qW&_bq3 zU2@`Je|JF9H~!%5=U!t80*I@O|iL|qL`S}eITXo%9#6{4uOP|qc^qYhM>yi z`lQV=Z%Jih(DSZN>!t@!Tg1>S?Y0;nzVR|%B;xx5ea3#Q97{*0jBAaUY~Ozr0E;$SqbUdkr>yb0LJ^GdTJ?Cb%MDAE{tvPC9C27h<^bX=_ewASiGS zQt}m#*G2& z_la1k(L3n{DTP>xz9#G-VGRxnG_03%|CE~uz~0%5DRYjpk|_^BxFR@pn2G7}1m@r& z6gm6%u0s6OT)Z5q4iGjRBJ9V-(mi80P1ZR<*ii5R_)WlrF4G~KnBMPLWoUz( zg`_j<%|3ZaSD<_n#AY$JT%O%?T>SZQgId2M{ymdLk?hJ&NM&6gK$Fpt1@QIrRo=B8 zYWrnJUgrNc%L*SLQw;Kabj?t@Pf$|OsZv@oL?o+DZXP4PjdAEj~I6UT`Eu3?S7s& z+Yo9mDPR~5zE~X!UolIC>a}B>*O3dLNvZ>R)o>H+v+FVbrRbQGi&GE5P%bYuxEh zO41pwQ|p^ZopD#f(=m)x3dwA3!|mBUa5*Ue?4MRI<@7#;$#Yc zIDQJvS1orIa3-_~SG1bFdxq=8@48hPO=1}657)Dh2QDWEFnkVB&B||9sN%dHy8-7i zIJ>haSd9MLYI|FY_HQSKROwuM9XM`FwP}Q!?ZkvC{ zRx9(9DgloPGk7eZ!q8AZvjIK-yCF%n{p`m8&#h8&*EjE*>-C+y*+LcP7`BfCq)~wc zfVv*d_yM7RTD!Tse3ahR&wwUvFw9D(L~66taz*|q&hDkw5zQ~ zzwCGEy}x8ZwThaM^f`E$Q1w+a6~w9kE`PO__o3YRWE(KBUuL<%1bwyO)GGuX9ld&m zyOezM6VJJUphQpx6ydsY)2@(M1Tou4%vwS zZ+juSwtnps2>L@94gld=lTLd-$^HBtDSA??m-`q%LR5%D^X~Vx3|8azo_@rSb*B6M zWFgpb0D@V0o`eddi69~)QvKJzBkt#V1@U`)>P7Di!`2@e-Kms2)1^%i{{$F>F@-Dk z{5jy&sI?L!Zi-*JI0MaJRm!{4EJ&;g7Q@HeKTJ2xE_!uQ?W+|aamYGYnUOy>`ZICt z#)t)R$uiRLe9S$k@&U&$DT_P8ypD2(7`|;!_nlBZ*qKF#A0<~ptMN~Mq8$dq+cWL4m;L zhow%vk~T}jR^cy2Dz%9vGe6bb#I54GA~{R9Z@*4OQl_uB`VHpK_aUA8i(X_~taixL z_B6UbSqj44e__l%5K(i8hm@y1?9b-Rk(nXkv5(`l5NrE>>Df-#D(x-Hg-w>Xas3Vk zSwFr#9bxb6E^H{*pBB69p1YEvFdM|Z70D(KPit{KcOX7TLzqy9S963#O(Vmy^P0T3 zcZfQ%JIb3)Pr3C)n=k2ZU2Q_nT~ksGnANe=&ZhmNqIT?f+r#+(#p zbh>AsHaQvcsj*Jgf&q-Txfq?@y%KC)Q{cg~ML>fmA0oeqFC1reDz@n zV=yiZZBgekzq5Z5E0tl%hHNEU$oTn7t za8^%dWMY{WmmF$_B}<23*0eumemuljtLug-*U7<~CKxsDfSOZXdbZ9~-NC zifu+{BL}?jPdZ+xOpIXoE|t5h$;C`lzfzM8RuYc&Ihe_-Tz1H{GJP1-*f*?`}^e7QoZL&N%VQ^*N?*t~G`y3I``#h? zxSC;yJHBXS%-SeR*iuS4;z_&;N+#nzscc?TYN5dWp>^ zFvI6GT8@N)u1=<)k>o9orGTUA4lzP{KHK+QOS9Um^m@)D#^f~@eu*ez#PuoNqnK`F zOqRai>}~VKdt&RF6!bJrC4K7TwF9fNCbTNIcT$d@Q9EFv?T{&J>*mxqvwdcF8;213 zGj*C;w7(qwYo?`u^0Y>?hrfaI-_PH}_+gaK26&^MvE?VTnq+1%rc)=j!RUBD|9@z!pV4Yk zK=J>?C0f1Uq^V_k=hJ4TykfBdzCHE+*;|pRCIaw0)!qXy z+lO^=q;|}NnX<3oa_eM1{(EB|pZs;5Bf+fQ?BwaeucI~6IuBX#{z1s;(qqh-Q=KbD z>Ujia3)G0~)Qpgpsgp^5!SN}!2Db&}mBr!rK)h71I2tf_X5n-9iaR#Gtd=rsSCH%`oZgL=H@Tj(mTZ@D{b5t zf`jqljr%Kqjblpru$x|`V<=L~TknT|j^F(ed^|JG>t;(v28;b^{6R>ULs6%^c2k{p<8 z+0_wKmRf4dO9ghV(L}%UI&hJj^&^5 z_4?XUauzNpc!X`_mu>KcZZRU;;vM{yU!nAt(Ux3Mp}9}JqLoXBNwZGaqYUgoN)(JR zP4Gd_zxb}*(1|>Se>7dFwH&)B9LgemBIdA?#pT2l@|PRaj$>is|2tnhZD(*|OOHAW z2J>|#vnI}j09%NXQ8;nS|2F-Mq6I{>$;-FY#9S9U12D;fxfrdi@$4thNS?t-@+abA zUCLF@JEy#K&85){Ij*oD4qk;>lDw27ME`U};O=9tbNh3RHaLV%VL2jwM?0lrQXNxt zxaIUEI**N3qmyqn4vAK&jvQJz=JJO_j5M`1I1*SDDYrSmD{5t%%+%7kyq zWG(!jIIJ7Ep&{qW_e-Mdu=E4d`ox&47YlN2Dntsb!<%!&l9-Gus7Jhb)U}asPpzjx zSur&=>9xmN!k%Uujzu(JzWG~UpS-WhWTxx?Lq+8=&28X2swY>FiOMmmIk7m{B;-!s+?gQXVzIfRiltv`7$nmOkW>*-ROni~lmW7!Sw zO3rIupl%KI7>z4D38SReD%BGJ!Yup)KBAa87G-XhfpB!19#=;ED3o%kf)uSZg&EKt zLIB@mIN}$f$$Ek0RDwnNkiypOqvGucUu$~xGYBT7%c3=e*9RsgtIl1?~7>c}8 z(QTUJW^yOC4>9l=axVPQ0OVJ~AFWgMPkYY7*QQr%6U?;6VE12SD_^)aRtzGD=_=mA zJ?NZotYOdcgZBif$WGx*zxYw99CAwAJcqzi@WVHWBd2PS8(vdw^-bvDmVS(~REV3_ ziI#WBvxdKdu}iJuRY}d#mIJmiGk>nTXNIzo)J?x%A`LjhCIo|t=*`3_i7E8vjDt+~ z*~00sfp$aDS^Y6K>upJ@thopJj^h;_&o=~_KFz3J($sfxDw(d&u* zk#1gx7BUE5)7beYM6}fjIc!hZl^vT#4=FFF1t-rO!l>(wMb?h142d}65i4e*7rt)k z*6}Y|UcnwP#yybh>@M$FJ=1pv--3!VhUI04m^k=hE3~QVAp;QM2y` z+U5N#53;z^-+Se9_5e0HSpUCj1dut7L-?aoW)^ zfTo$9=f=k@IIyou_7U z9*H0e{b7iL)V&TUEA_r*k}f5SZC-N_k35yu#YC(7T%n{(>CK_i{-P<&=bC^yOhh#D zlb8RRSHz{2lbF*^Mx^NW0x?ID+jp^Nr+&7}tPF^fGu`(mTuLrX9PA%=oaluOV6*&u#7P1X0h&iCk?^1n_uH!6GPN3xEA=>FadODsRj-5LTHi__5~Vqr+f zP_H*tO_FfFj1O~TruB`B5)x=B0xf%1{vR>EjnFXOn-+EoKZ`>Se5CSZgZ5$c8HR#` zGzL$oO3Yr@)eA8yFqh7BDSoArAIEDyRMob9^EXg<%9T-eNK~qfqyzU^&D`pD_$=Wf zYc?qrIfl5wb0i>0dp~Z?L0ISM#jof3!Wq8HCJUCpt-GI~?iuss^1N!lz6^g%lWAj+ z&8r^dQnoepv8DI*hL37weWf;|SX!?iUsF#!F*sWTyI5cm^;S;djbqGeThg^=jyJBd zhPmU4TB7gQzdYQ7yM82$xqeywX|_mE>mT82y(Rye8L0=1ZSU6B7RpP$FWe}NF&1UW ztW&hV%#+&qWAFksh#b9+b5c$Qc`Q}>&?bb`4ii6%>PIaeSV9o2N$T_qUOa!8v(z)n zUtr4&b8BG=*5qYu@G1IWgO2)h*ee{ji8)O|j3=C@tKG3P`$duybXsqu`My`Nzuln2 z!+Nnn;=tHvhiFU5vemx{i;_$aw9+1OmFa8h>zwCw`SC#RmsqI7>ZqO}_`CxNfc0ACqi3>4WnwbS@gqiAD3|DHQ7gMUa~AK#4$4lkFJ_%P zl#WRe3{|s7GaZniz1lYb2wTc|X{@SoOa?A5bD_xO;_uXpUHY~`kQW>bV(!y>wpieN zr~gII=e{wo;cRgeo#~b>{RJ~Ex&S}Qx1*6dFx&w@7`eD9ht+ly=shj#e?JwenC_HSsiEMx; zd(p#3KRd7nY>3dOS-xGFw5QeiA4a(A`kMecC$?Y3nonvOhf9X`#V9$P&LICh7vtry zw?dAAnUI8Kke2@O2yV?9AhD<8hr7aqvlpVv{|q<30bD6`QbPL&14lB?w(fPC<%f?B z3G}&;cSF6iBw=))E1ybG4WNz_zZL9!SjBG*-0r-~J=G~B|MOjUs3dx^BX4A1OcX+q zWBt_YxNCYf%QH?AGc(g!xY_IDy@#mgA0XR*5^U}jhKEyhBO^!?zGwURMybFT2BP5CM#!>w z-Y}2Mrgs*wGret%uX++W^wHM$Q2_Y-bjlnGweK#kE&YAD+?5hi0h!6#ga1=hR*2)R zFGoW*S0lt9A0Lj?{}#1Gg@gQQug`8YGLhR3?O4zY|2=Mo02XHDyg}KN}xvIy}|t{oPd`B#+lygFY|wQs0C_ z!=9g_PMb|ql=IQjnd$=%SyvudlK*sma{R--`8UC(t=4AEEaHAC`?}50TtjAm1ZT&X zHFFXN!=IT=hrcguMYG3?3a7kv)MOaj0}KX12A_0{xvaFWS^ynv2n3mu!HX@QuV7LB zl@W^GKRmEw*41$xoFaLrrg?SBfzTAlBUZFguYfVK&l}}?D@*40u12$dUP%X)V=z1l zejx=>d(i96KpXR=D2pa>D4wD&mO;DMFNH4kXPZo2F^6Z4^uET^CvrK;2)7MZ9b;dy z5}kI~Q0fZgd8XDC5@p@HQDMC39W3!T=M!?XgrA8%cyOR2X2n2sNVuvasjafY3FTXh(;FEdsvIlPg)gL z={~;AEAdBG%v!a26Q`3Mlc1oX-!t#ar=ecL^+4-6;ZX2|pa1KM^n_>nR>~oEJa-dE z4n?RczlASGccV;s?zBnNH+9MB94!pI_f$(T=&|18fYwIgy+a#TcPv&Ti}n0b7L5XW zTJVMV7Gj*TYqsib>lEj_%VnzVSEg>ub7eOkc%5)bsZ#t)e5QpQEVc8FfK|*@dP2Jt z*beUZenPDDfokI);jm^FrrI@#lTK=n4?cff!7I(RwK6NG%F3t`^r88L`<;^i5?mLm3>*)S2d1;_-ehi`=}DAMR+U%uk2))XPi6Qt9z$hTJy1PeUu?bp=(t3N7Cv%F@{$vc?n z-Dh8`srrsben<}SqPMBR!&+C@@}pqNrR=Ue>co89rZcJkoppE4p(On6+4#IW3E-{pCa8)QJSh-Q+~*kKlZqcby2 z#!21q0_Xkr7wHO*9=U0PCEJXj91NJ#eXI50>Ro2qW%g-rDWg^P$#nQznMa@A7RDQ6 z;5a9{2%)E3#p#loAG();hA?)F|45tHo!T;3ZFOpauD0teax=y?M|F;K?_-lY?PC0w ze0#xG_yB|b(hVgKG-e+BuDo`{$}4%Jul$83&W`I1x{$EGRwm)M*7NwpahA_wFq!4z zVF#Q;mG!=RzBzi2fyT2=`1wN&VO&Z;WjKX9wm;8A#V_GLA6g*NDTcDf6ylMMWICTP zFG=IAU$)4Xw`*JCX`GqHrjDJYCJdm}e+J|V51~?p*@aS-v&>iZlEJZmK>cGm>er(7 zAzh~f#3S>KL?ZS=eSKkxysXyjgp_ia_t=f9xA=l+G6RO&s^|q0p`#g%EnDorTVnX@ zbvzWRfQ^5XpzE|8`MMe?`U=XkZmEPd0>G(X!>3+n zFd&+|;xlbpKkx@xzsm){i`z{1w;QThTR_UK^_i_g;!9I)``#(|Ll`+$YJa~Y06Mk}}Pu|Fp+yvYWs@KG(6tUNM zf&#{w(-(nKd9IF@^n)&JFLK8BMSB!()uK?c?PO`pHCFz;44x&qr=K1jDn8_{=e{Tx zrsx50x9KNNjASs_`FGa+O-` z6hFJ?>yJl4K!bjQK-G8hj-4FJOucmcI~{I#`Ad^H(qJpDQlLK)-(;LGkz6eZOU~AK z+gM+s^&D>zwU`e>1}mgD6VRuR^Pk*`ezGBYrP6+$@LQlEJ-5v2i^-5F6@$-c^wWkQ zO&E%s+@LxXB=HNivvi*vw1mA)xpZMZhY?}t-=hBZLd|IbzvB<~pW^uYE(h!MkjWE4QZI_%c~H?F5^;+=x<-7~i7o_V<19nn+!jJkjv(0&XdFNSZLkvt1;Q z`2EI3mZ9WdrOYK>ix5(+8>=hE_}mH9OWGjW)X>W5dYNG6oGy>dUTFrxM(2;|WTqjM zn78{J3ZD4I@*sNyqy;uf-$CEViSapBa=-iesn>~yp2@9(H=25?&-ogm4TGK9>x%A1 zKE64>x-{gC*I`o8pTy)IPuW;eqWzb>wamhec^?XQ0?*j{2r}uvJEn?#7ukeWszqxoToLStOC>A~pIRcphE z6qi3Et^x0-e56-@=D2IPKJQ8@ono9~i^=B-YF=h_zZ#tdj`nxmk|9{()S}$gHZRX$ z7wvKLBa1viru$bS4o*1BsIejEN>+jC-X2R74EX|E=Lc11Z!ymbNZMM-#=@t~dfIN}fU@H*=Q*BbWm zuVmSnT=0?8wED?XgyhJg$=Y;o$yVt7GfwRe;z1@N9Hy*p{VuHdSP=3hCbDn8+ zsyaZJ4B)e;`YhmaAIovyVJc9={WkOV8&JWW#_LCf!$IYY&Ih*|l{IB)2c?sGGaQmUxh; z`)!*{(ipToBh`MDaIhG-mm&2RD(ym=!kLGTkK3x{cOH^c_SSESQudmA5Mp*hisG4+ z4ogiCIkJ*(Z7!uw+?VPzEpQ~+olfhD4%_*wN&mTl_V!OcI{QE`55{e*^*B^?+&J<@ z%61wwCShYnguOS&TQ9|^Ubil~JbO^9tpt5?BWCQ6$Dx~33PSaNE^EfqHyyRrW3E)F zb~$}oaMXXay4vOSwFLFq0~YG3u0KE3=vIea(@=XTf28X~o!9UD%a#Xz`D#?t-5Y$a zr1AW%(%sqmxl6?sH});Oq(8sB?z}7`jTcvrnb^W0$LwYBizybRF?f5@1V+kaW38D$ z-r7S`k>@Dc^+wp@635@Kxzn>WP>M=jlBD1wA%}$rxx`~Q>ybi zs5P-re>Qx1=XQEbC_$8dmb>0DaPnqFHbwd2<$tLCb@!?Ysgc@UN#Oj{+mtRA?S|5T zFT4F5x5L`+xlpco7ivH-BDO?hce{JWKRUZBN*DBiRsw5E=SBL}Y}0?u3z@!~;fZc% zSoA=j3Z=h@!>_u&NGtZ=otLV>-0(q{&tszdylQk}cY}vs0zbe}%3g`s{99hlcmKVt z$tCbx36ySSE&7?!0}tsyN<$s?N=Rc%6g0QOpnE7*Mc?;i8@X@Pt2F=_5;`tG|1ybOo8fX=b=@1QKP zI+gANG7<|X#Z)^Sy%t%}M4HQ{Qe~ZS#@H6BG zz(A;@8KhtlhOA~wz~5-;B^f6ZRB9@2F`$Q%uuZ4^V(e# z^v=&K@^oxY#0cz6*crM2U@6pLFN?f6wdM_0P{nBbV-}Pj2)sWAB0Tl=GJaM5jb*Kg zM%mo^XBdwOh`Kn`xL+Oe#_HGG(52~JdT#-KrTyFF8PVCz4eh=GizyP_nGh;YsAhHP zoa(1^r)21Z95{4=&v`^P($kQNZ~2J{6{NBZfzNE(_?5g-7laQiGT zym{te*mmhZo|_Bg%U%IL2uL^qyirGsgOZdW5e@8%+J#s>N%zUgP&ItFOEI%fO0W{# zB+sE6lM1eVL|x&xt7Ik~k0*D*SNON2cdk^Cx7G>YG?Xdpnwp9Ea-wh%^shW@+=W7L zJNjfF;-sD0TVO8StpJ!+#ti~KZ9fN%Kx68LK!=~}DzKb~=a=(>z|;O)L6qCZ+KxBV zU@ROR3)nl;4z8pQN!9!Njrn=G(uvwvult{1o%w4g~vG7iNt(0ZMOA+Lbj!Fkwkwz$1C3 zmintR^NhHjhdFG;@Rn??)^0x9Sy2ypbpk>huc?n1be#R{hdzS=Fkwd75@6)%c&-6o zzE*K8pow}80d#GFy2%Z`j^nUdKLCs1iF;M#_tL2FIG#$Iz4qMaA1r(uU^rRpDS`&L z!B8|8%jH{5l0@m$Es|!$^rPL@8v~(+#xM$p)!~){Z;Y`oQS%KO3({ z3a$ZnIP)2TTMj?HnuWV2M3J4lDoUb2@Ax~)hW6^zfDp!hONkwOqrqhMvgG!QL&qxY zVD;kS;y(B7?im^yJ>vvm8Wc1X_^1zem%qAu;_&URale09H}EtabboFS49{LyC6vG$ z$VDJrGD!cA-yQJW>nc)wkS4ZW5kSFRCAts)aS)R=BQQJE$mGZ~-58tp zGDa~U2A<9vGU=AP`9c5i`5V9xU zC(N97;cz#%UB|ku*&&Jot7r&WIRKO(BjSV$({2~<_(u@;cD&}=2^tS3KvWhSbdAp8 zWTZCc;>qBP(D+?941dBPHJ>Y6+bl61?)D$8cA=5dB;si{hrxwwyAt z9(w?>)RBmNi4<^tw)4a5N|OFPCc99biGiBeDS-MQ6v3R^@Xz=|Ai4(yIqKs+@PB_^2aMZ`LP;&Gf(PYKXFU^>25u2j7`f?GXEkJw{mx4Xo3j z|1dliY`X;MS2xu0E%F?=bl&?Uf_l<|iHy^*X+V%W@&6bT(JjoB-BGceKhgXiIDdN| zjml(plJx;VXX%RH{V5Gl=Z9j1TPw1hoK+sa^N0qjsIjdLL_knEG1496*+gg5rma*9 z*KaTzeo!=KRA;`*tFrUL0y9e$z&aj5FMKME!7h`HWoc)9Z!;e~vT%3>5JC&U8Y+BK z=|Y%xd(Xoioj5G1SR)Uy_RCDyj9YK;>qh)z*v#$wT>m-vhhM|rpx>kp`_B+l*(SRV zhCk5U=DTL#!I!-J^NDCl!EQ7^BhT~j2Md3ORgi8Pl7fi6O8sTvN;|{j z9K~Xop+yBwbn?HZu6k^4G&)=?32KC<5Ls>woCyyT*T`2ah>_GWo(k>r*l`AjFu_s) zO4QM!?hKrd8n`~FGjq*|>^SG`u)oJL?O^XVfm=~A`NqG!1NKV$Neg40M2I1Oq7X?f zhXsAiZWyz_Ll2rOD!3_m(f5*EE#1EVIIfn>WV8yz=crbQINK2LxuXw{nhX3l9kM9g z_g)r`JTQ~VfcE_cXVx51_k^a5wyAoiNWa_-xno5h=E9<)UFEYN-N>|yrPQcZQ4rvc z3dA5UZoeqaczmLttFy^2nEXo+4S3r`caTIoWtJ}#!2)KBGCVSUqtCnsIX&el5|7T* zpHg(8J6E)q1xq#%z6MwyHS#=jdae3zmkPwMD3eQxFrBHAL8D-*sbl;a#BO@2Gl!Wq zVjRr<#|66pdruHeL{ESVQ+F8}5i9ijoBoBC7FjS481={_Lb4Mox(TWL4Q3<6i)jW>CW+ z;&ebCcn`<3I#px%m`B0Vo3A3Gq7>}xloOMpBS4h;mS^*vlT{j{Ajg&?%}RX7aS|hs z0^gXKo~v>($CjKW&VEt@-_#LOBdDq$n;A6+@1QcAo#dbm9EdyYy=@dTC&w@5VdsrC zqF?^Ve61-)hTu|!8wZwY&^l-Pr&Z2;IsC??9*xIPG9oF&nVH5wY!aJq4oE2Oq+0oY z$ls{RS-SS!u?w)@n3cCHok1wF+XuJ#apS|_?_?9?bi-YtOT3)KPQ-!F+i)ih=;^%l z&FY#W2B%#_y8=9`qQ+F%+dS@K5vldx3FmGB3hXOkfkVSCb4G&u#krn-gs8D4o_B z9n9Q|^Y@h3d0MTB2ZQwq+4k*^oBt;j&_P2JaDHIwyXkqS`w~VQYi+9>UV>TAi zKA6Q$8OylP$t#=z+Y>nP4f<$NkTYV&d=jypWW1{{1cB3EJ!zH~5oF`PB{tu#wiwW4 zJ_qQae)-kqvvPcE!Rw2Q%1O0s?w16gwwKb-jJB6z?t%4Id4wu;tTxr3@evFJOqRlV z1%7CK%PZ~gonJX=7ZB8y5!pHj3N&xcmbA)MB-Nz^CBwpbptp7e*SG`O40s&t`Z!ck zkODoShY|wqAv57&y0UmB_ja-4t<}G3;6hnCVIVd)4c+wNmxzvZxj}uBnG^(9475yA z#Mxf;m@=w!K-`-~Hrn;&yt}X1`U}`~Koy1UR)#r)A@fyaB0?XKn#9&6f$1<)MwKqN zS2=goK}K3{!9zonq4gfDFlEH1O~y%`Tmg5cB>NeM|0N_Q#JOlFSW%TA#eW{2|34fs zE$V;s5V)DTas-}A?SQvO=I=NP-G3km@^FLw;=h@BGBkjOVz;iu0e*cT_^cY9dRyq>L0C2+zm2j-=$ykGjY#I?A#FKWiUlF*sDh6V@8PG zP*B3>{A>Zr7@^!&K;s|14R`Ed3onY|nuBC9Do*7+|4*usN~-SehLYl)9Opmc+Elz| z&2ykAY=>bejirwC(-6d;>450Ipub*iwEHi+o$Q51F~GT8LAZ!{L%{ZH)()l1sE+@; zzV5o^$fG)D*Y$O7erHi=F!2L~8=|6x<3&{TjD5ln(H|#7j%#VjE?m=Kct{n0EA;ba z9zd$GDwWF|Bp&qoms0hP4xEt!GeUh_jmeXwVKLawBgbA(jnt*x06(TgkytH|-blR8 za{ia$r1!FixICMLge>Y#E^WW+-;+ir$lBdm4%aAVPweG@b|&4{q4GzD>3^rQv|RCA zGhC|MMP-l(IiA~Hbp?>1d)j~_87O<2%&Ltp?>w8{X_TN-!cxBu{xy+|ls41m(XjK3FP`ssOQ zI}G4dHJ?ZN~D1yA1|aa^kURaSn3F80!d^Ci0&#Biu&R3Kjtdj5q$=-vu&^GNO8$2)fzDT z^B;lh4vOIc4B$!vm6{0WG2cO3?528vrK+(j<^$1md){wI{$rBDPM%Z8yP$N9np1a9 zmInMMou+%~!B47slBF~P@)K%0?sV}6|9^$}hE(JY5m1bb{RC0cpkp9>61?E{H0B&P7u<1vkIdBN8Yt9J(apajY39rB zX4KnZLd2@^4`9(LQmub=rV-PX9g3`_pa|(O+afOQs!?2<(UqOq)>9?;A@YEvVCPT6pA)6eK=q;rkQr9|Z(*+qlC|a*q4=(}iSNzW@pi#n<26|F`!zGv(m&K`bOHKnflACoY0Iy!Hyes=i(2 z_5q3D{uCh0XXgCdmk1)wHVu4I<+Az;5>?5oa<{e{(ZKGsykt`8pQa|ZB3`gpt|A%I zuC$YtaXa7vu13w8wE+z52WC$l2K@OMxWbzHrMf_jS*{|V1;sGJuA3%$!!qKY0|Q;CNpmc zqN2}U12d~83zW2U^#7h7o1-YXVy8A2WN=QgapKcxa`e(hlnglPujK%s12kI5Rt7bK zXAUIVQzd{u5ZGl$JU7L+1(R+S{awdxRH4pA?x6{HhW;S@JY>*kD5)2Sh=HTk^=0Jn zYLJr~2%B30Q*D4=%}$CX=oC^01`pN zDYsNP4mOS(oTubJDp5tE<3K{Ic(|IOIs3odT)T~fW1Oq&@R$W~J}Yf5et^uxM@j8Qn6tU`mUqBF(-jfdaMTJ>r=@^SbQm+eDj@LUj$!-fvB}~0j$tkC;5X=^`lUd{>o1`v5fX_Zke7IOIhxVlTWS< zgz;~*s!v#U>EgD~j}f&v*DUt&eOkPedf|@}(my|FB>wPPoBFG!oVO_!Ptsk9yMmWS zta33Aj1=;S%OuyV_cKd>XWJi}b*{Ui5ZYO`tqM(6J|k*}GZ@VHCw_`5?Ej$AHh^Nt zS|Ny$zveQr%BjIIg#N-gZJO|-=OPVVYt%8ax%1$?^d(lGsPS>$aZLn=tT}`%6w{}+ zD(E>?t8YA*Nt|P=xcrzidvaL;(Ag#naMc@$7h z{vc^G4*0Iey{qR!Ve9d{``px1^$d|747(Tjo-9iw2ukg^W+j00&!)11_kVw@te|b|l5>q^p$YS%z zzE;RRvrCU0xH&}|E;6a7ReI?UsYDYcu7_wO9;VTfYNc*O=7y{Y44+^9{krAF%lIwu z!L8-)lM^|$?1UItxtpGIQ?5uNtZ)6zl4Yd+`ylDH7|B_MUc= ze4A|_i_noi*KXtbV! zgfFG5Pew)-1QjY-qN(ur_3q+_A)U^3cn8)lWr_0Xx^F~kvM+7IN`_PIF@XwPZ;mc@ z&qxL-(_9~74wep`U1=_Dli6It2g<~NjFCZl39~uMzctyk`8H7VYqq+b9UU-)GI|Zn zOkZgh?pBFQ7uK8V=2^%PfKT8I@z{q2(K7_9Q*T z7BV8z<>h$o{gIhqf=f28yUcpZ*6#*n%ibvyO|M4yliHz0pZL-2>Slnk-H(pXbm=N> zR!)yL-_(T_Y=FaT%SGZZqU9rL*yj``vGE7$MV1U$HBnzH4`mP zEC`)>V$5cBW6`3gkTkTvBJLt2vWV@Jo)-6-5CN~!vLfk`mRKVCe9f;2e!rZ#HF{X* zqzSOmNip(_DUshZBB%<&l_=LqzmuW|oSIRjD-lX<_;by;WrUE|@q{mEYY%vFT}ZY@ zGD0)hnYo0Qwm!ll>kiAwmH}?zF^eKi)*~}C(VaJ5q-Pvo5THuzehQGr6(TX ze7daDrMhIdxvEmkY$-whQ-JpLKl{O*s zOmfH+DBCp)Cw3}f@Fl95Wy2ty>AXdpom3;-EtH+<$m^+S(wPVOl=>T+n>EGftagc@ zF0dk(9{b(k{8TaNC(p!A{&i&n#q9Q?t~q{Qj8E`D#uFhPoP8CdZ2-x@Z=sI`Yn5j` zgb014-sgbVn~-rGqoZqfvePz~gcnO;5rhr%=4hCA`K9B9*5pnv((pH-W5YLf{F?Kx zC0^3l_cCN@;8%J1^{kbL?$<7+n}FmpZotdN1z*}sP`;HJ5Ha;m6wR5>Nc@AMBWh%R z7lrpcd2&xK!?CZDkky$NXDub2Zl3mi#HA*fiBBmDK@JMz?U&~Ktm3ox{5hcb!Z*g) z*~Wd$BOpdgp|}87YOK2SNZ)|0z~b)sYt3Q81Vm&0SDSX^u(@3K-bhFOThCQSW~Qfg z*EX$X{%49_&bu!j*qH4+u)Lhx7VL=ah|$&{u}d$m+Si+FCK>7>W<;6zx=1XcVJG+e z$ZZz499jr9A&{c!q`TeGo=(jN*e79=MwhSxhWUpQkQ2d~4`HQaYbIRJ@8{ax!jT8u zTm6NlgBALki5YMSL>-<~+#u7lmHtv~;j^ZzLo4|dp?b!B5w)*&o)9~+_-YWp<~7BdB9n)it2`{_qVPr*lEV5pK$@U-n4?Xu6)%x$8xNpb^on8@DAk; zBaP--F3bJ6t#@7|1LtG}SC})i*@^s7fs@IR?^J_pjg3704Mk>?WiqXKS%U*zQF^(Oy@cF5*wS_qq`_i;04Ibjf9y4FWMo9b7sq1nF`n66fXWkD?ypE6 zW2%c27vD8+Y=KV#z70cN-ZNqc!XocFp)Ii!j(cV248R%gswgz`plY7`<-R$@=CI87 zV=T2dtN$7p9jO(o^laT!MTN)p>%Dxe2|Y-|pj z*cjO+u*}zaKgPlP-Fzf2qr_<3wO%v)ICKlcaDOlXynq zz2?O1&P>Tj&vUK6B)BV>n->f0njQk_%c!?tWF-$VBu>yNau*T@a#7CeQ3X z7%z(#<-659##_r*447D-)^VEk-##*e*?yt}K3(NQOZE$u4(ppmFh!2C*ds&Mxw#lb(7^`1@5$p zmx!}VYJ^Jr@h<;DA>~Xq&g0AxA_~5>Hhnq73R|@c0+X0KcZzf|(epJJhITnY*;xfm z-41XPt*~mDH%^N5CQhW&m(0uT8xS#h4DQ-kX*savf*_Pg&gidT(*%(&;f<0DE_z6m zprC6DClg`3T-m)OdDbe`GGMah*rbYwEmeS|hDv_Lnn%VR*qY%oplJM#za!#k$yPlE z)~g3DbA69u2!xo;GWe$~nC$qJFCD}Y7xtF+qsB~)XySW4p{xgsmZ&eJAI(BtA56bG zIj{AP#%SAbHBNN9*;xrO!d9!(+T>))gw_#Ymi0~yLu_n`|EW-A^wNSRztc3k?pmDv zvN&A6$`@ZW8}S{Z8Xvc82U~fzC9ENP)258yG{#P&+)z9sCT)0QwKpFZK5p)+1PCdE zQq0uLVW|>;N+=eHf$TSQJECFzjAUIL4RaUR*6+uN3VV4aZ;^uS#26aiBJ=AM|MnKJZ`Urvjf{5Zq$pO-4z&RLvOS0;&Qg5Ck z8_|88wZr)a`d)jvoKQy`Tx?NzB9|^^56y`Bu~}J}cTm}BTcMzz9IdW95A}S-UOPvB zP}t>Y8mzcfJ#;Tjak5~_H9 z4~ec3v;`6eba9Uy(O1v2_pknbPCGSvqP3Q!BZLSvNPn{ zTNy2MG7H{0{7vSPSr}4a5$!lrxPd`uS*$;8pY3S8J2@=nPnsv1qB&7SB|LA+)`(rO z{77Vb4b91mw4}))R6iG`f_RuB(4OoCM+$zAc51)b#?OYds#~RHUz=z-Dd)Dii1O=j zn5K&f^O!$(s$~t2BDW>lPk$uWTn;un6?FJ?A=bI}M_{(Ppm3J?O+WS-?bYM3EmHYL z`sS4gpM*UU`pp96qHnBBzph4HX>2G(U_W#Ah`m9$&-p3rUor4cMt}p-=zKas5+?dc zKdrZrFqB7bDm~~;x~ur?I(CW2r^cJ{^^ZuKmD!JrkDKYq=O1^~w3mf5X}Zq>?o$8p zaL-U83*6=?{Vc-)eW~`bB!U9TwltC&@X_%!n1||`*7mZgS(g)OrWm^DupsxkfDLX2 z9Z-ux3^><*ta~v;qH~A8?YZ<#g8@@Lc3ufBWRQx@-h)ovdPN+a_!d?!k@2nX;C^0? zGObGkN#SB>>mk&Mfqmi9Ye0zJ$zL;#h^!Q?D!X%|-Y(|z=6sS2?4=tCOZpa8W~Lyq zHl7fQT;~YB6|p(XK5M<+BcIj(p#Uz$*E+D3yGT0$r!6bLcrF5AL%rT{pld%P~>h$AMRC=KEM86XL{YZ~o4LJKu>%c?t`augL1O!FhB& zKIfP=Zv^5KY zd$T*!bwop=N&TjxkGT92N3%92Vn~sug&?*wpY{_OM4(4|}v6MR|*>t-fMLdr4 zkng>6a{tsLY2vp$)q@>Nkp*P}ds?KNCiOLr5un1-c2DT!9dx1rz|nMduFbTWi9TH4 zm$9*EtMye#0NZHD4$>;;gV?B_qqnu$L5)X3`v?|hO@TFkNAJHPfE33p^^MELN|!L? z+Tuk0^~{HQoy&Fy5@VbZKavrK3$okS--q%L-%RS($00J`7BcuUTUwD;t)U+PnSB|^3dn_v9{UL zS^0#OFfJ>lzCyMz?xeW0oV6w~n|DW6yJDQyJjm>>{(5f*jD~gg?*aOl2vFkuHYS)` zPpwy)mJ4NgADTS|cErUFU8Tzs)kls{T01r=lDq@F>4 zgvhVo`BO!|KK)EB`yz{#Az;HqzB`^II*KX=u!Y@G+cdgKzem<%Hbr?D!hVBO)LIjC zb_EbrdzLfGKXifurX!X}GPjwni?3zNanp0Zcl511%I^^Ai7hp*J42f-&e$F>{^`|e4N9cXs9S(^!geW0jQ;>Q{V#6#)+>a7D_ z&cUb4~^)1?eG@%#XWI$?>f|8)-g> z%)E_AK`=-W`Lb1kC;ZAw(@>V1DTEWMt`{&?$6js1q36fImw{qpdW+15a;{(DG#A}! zA|&QEgPzOrh)Jm3f!gDkF-ijy{!MA~4Rsy^)o`wjG*)x+eEEmj)vZ|thK(&51qPY* z03p)fP4g?Yw-%G#_AO>fZxwIIUj~VdLS6!}7&##_8U%Fuubqy|ea#lG{nmy3t*^EW z_qrOAGCK~>V}bG-i05b*nQpJ>IC3HMcgz$JCPLS(iknZ5ix6kypy^jt3_>sXr!hqZ zln@eG%5#nOEV-Lq(~DS|++r;=h=?wtH{1u0=$;BOB<*VbfmaYyH1onVEUl2nYv zn!^L-TSE$DwG0I}0wLPvjmNrLlV~q-t zX~q%5-+TmLLyZpfUdcWju^OZX^FLMATG^Iyoc=1>?PHJc)fgM!^Sv`Hc~^~>t$DGP z!QChs$3lYSjcJ+Z->cip61!O3334*769USu?osT;e?tPAkTVfe7TchrgsDBX2TX_e(cr}>_@I2U0^oGvDz-4@N?;^PGoQ(H^* z`$8^9e6<~$m@p30|1oVRB3$J*RYPFQyC|Hfb-%^xR3A&?m@LOEfU@;F?4^dcwFs7U zS(f7Sh-w#6`JY*B&B77k;GMn9GkQb}>WviL!GV}Hp!??hL&jvwi2cmOAxjt$POpuh z&brvFBh<+6vwBdj3A-b};9)6_+iP5~1n67dshltCq$J;tDZ%WZ%6z| zW_)CMg!Z{$&uX8jTcPd8flYZ2JdvGUSR?vAvxAc>gI3^R@5?OFrfPB?h@lgIy}DR% z7b}r2v&y)Vq0)U*TdxSX!nrcqlLQmYK~8ejNBr_VH;}5%`$mWBXBGbaO4+bMspjUF zai698MA?s1@etE;6vJ^&*C(od8BKzHavb=PXIRvGD3VL`hzUh=|aHpCXmcPHiwDaa`$~tB*vPsEweyEwAW)yr1BFPiM6$%dt^$ z-E&(aLdyHTgITcf>%C&Ja_3}9nnRPoC0A08f{bemxjK=xK`A9@QSb{~_53uk1`(e4 z9(N&zT*(!7Oyc=tp9k-MOx*m9`_LXoGXmOqiG$SLIFNo;Q!xzmnHSvHO-+P_nfKNA zNyiyj1{;E(X4VNs7;tbihb2DmIn-Hgm-itlDDQM|Al~dmT;_Vo2t1i~W!RRT#YlDN zvM-6$yfv&WypP|PWDx$q(nA$Q;C$I*?(@ghhUhs(hI|S8oj8I<@9Ui=vF4-wyRgfW zcl>No11o%UqiOsE+C^_+t3d{jRv}{Lc<^rJ7?hQtLOZeOzxVVM=Yf3W;HEAxOZ54H z0ENi0o`RDD!-BBQkp>L+F!8}Y>qzhJ4W%*|ubY)y|Nm?6%HyG4+rCccsOOyI>FAtf z$#dGYSfVT~sG~zgBqb5!5G5r0ZnSYa$dPO*OIedGOSYL&LJ=y<3}Q%Q8%%~VW`>#f z`u)ZX&-3T=dH;Fe&$<2x{kD6#?(4qp>;8W4t8BuuZEJ3D{)dQWZYb4l-_@08a(u2I zUv~sgs!nsslVJEc=B;Ng7&VD)q1Q`pOZp>Xmo*JpqjySQI2tL7bv+)UoUOZJEzZBd za6uA^;D8RG)f08L7p?z#reyg23~XuV9@pI~WAQ<=(rr-1vFGk!+fZ{urb|y0O(SLP zUi(3t$1L?^8tgOR-9B40)SSshYB<`$U{cvStnU0t)Y${a)O?mnq#WNeMWW{8g1b0+ zwG&;#)fwck(zLD?%d5sQHrsTFN{x{{L|FZ@KgwBkq?D{KJdKVBhRg9hiPA4UE&F%l zJZ{~o+aIx^RmIRsmKpWn0I?%CjqUGRIaOt_-IZA96K_JOg&LVz&(x0Gg#FSB00cDy z+G5FHx`BQ#-GhUdN^7ir)|>e2O^9jX?5pgN$PSvt`=I9k{GjU6OCXV9w#oRd6#_9 zP}lv?_DxlsmW6@wIk; z!v%~#^NWuKTLmF$BJrHKui9T-~L+}bB=Z|myfs>)z@ zJ=3~?NVwQ>DkTIway0OMd0RYC*>aP^!~cnS;YbH}ByxMyl<${l{Wc7@uxv4o|f9!#)shriDuWIHYXJc&P?T@mh=! zDI7K1uw54!5q!x!-?kM$68%A5fcgCs=vZFYKfdAzec>i63E)KZefD-T1;2Wc)WfJd zwv5ttyzfgT5Qz!^DT9KT*FQD>oEyLq$GU!)LU5V_>_|=eQ)X45JyoBrrFv5I(JGOU zBM5LvGW<$5EloaH2qm_$P#_BE!+I!#DqH(@LFF;Wm%>)d|HC?ABsOQZsXgEl;3#a6 z_v|ne8#^x~52rf0W8VQT^WJi4|Nf-()&XEyP>sVI5R6_;Hd^*OOU%s5%mdezF;oOC#fQhJTnE)cB_#jP8f)N@ zIxS-q7HK;EF07BzQzL-Y@F{P5eq1`;-Sec^j~IqI(zGbJ5w0y!3D3{e&j6*sON|7& zSc}pSVODc=TY78G)hkvhjc^r(s$Ev3DLEj(HNIG#7N0=WPQXXe4e%C1y;BXw7F1Xy z1XZ}Wb(>j}{M`w`qm`T4cHo&fdjjZ&kEsTj$Dr-?r7xpeU5>!f-diY~845@CKltJq zi3jalgpkwMYcFHg#@>OuBKWS(>Iup}0@LuPSo2$wg;ln(*vF`9*C=Z-Wf=|w42i!; zb-%2ve4^9pKQTQ`_ z%5NvC!UZby{x?QVGJ@yhgWh8^W$VUOMHfGMc*EJ+fE$PPU2!wHeNfvVEdl{mom-Fzo30C!s?-v<)Ak*E1Q$esiiy=Al zglEcn^{|a2xMk0TA%L0|*VN&u)?UA>8V}^AfF_|5;+VMsUXD_yZMOP-gTV{&nO4yn zW%(bs2+WDa+whp>>}4q}RRQGAY&(1Un;+T17Htae{Syr9vxN_N`W&8oN zF<^2FI!UDMYQ^!uSAqt!QBYVm!WqrasM6L@nmdVLl?76BSiM|yv3eyx{t$E1Te515 zt7GX&o{n00B)l=^DImjVhjDnUtd>fJ@%B|L1k=TDv)vuf7>MhYNzdy=lGt4Wl&JnWdK+&jiTxu zi3Q7Wm*-XkC5xZ5aDA?lQKxRe&q@N#8c{{*N&JP;$(Ojmmd(rHdF1wPA^a}YT;XVE z))d8V8LIHkN#qm-l+IXZ`}^}s&7H`R@Tk;S{L+r#ERklsjf}WsFp(%Ok$TUnR{;P_ zECw;e4nLB+z2^qP33`WE?U3aqWZ+$EtMUPB4G&*i2#Bo^={BW7AcBtw2ds#EW*=}) z7LT4c54x_ZZ20|jo*1<=&(d?gJ?6n7aAO#+$3HzDtX1!%7y0#R%<;(z zLm=@F0q6g872|PmzBL4)_`KT(Kkqz$&>vU(kwCgO^VnlbP7y`)kp_*^vxbHxL_|0c zlrtT@BR}*}`SH#&GkIAL08zY!D2Iq-vcC@-X|T|&zKm{h37l$&mRzS) zi`?!0C7RlX#0GgfW@nn4wYmu4np9u@T4*p0(cf=GB48J_3vq>>n9%aIK1jp_*tz=_ z_)9~p{*GeaCHgIs)UWzx_5!2LLX(}nrNhMZFpfj+sY*#`&aYhCe<|6B?0q7f^r7Te z-OZbyr&+BF-@WC7sN0jtXK{JY*4Ub_laNra%yzr|!-g9l9(fI>96s&!-B#!B{9`JG zr%IkgeBV~`ZcmVC%MX{_`ylnea9_a4^e5)@@#J?NG!LG3E2rtx!vOs7HzMx_{h6BO zQ1gTa#$?_7Zvxq1U<+>{c)~qN^~XOq-wBxPx-DIR|GCfC4ra2E7d=+ry9Q8ovy5^5 z{0CbYlT&Vhi`?^w+8eH@<)w_3O38v;0H;vq(UqJdst|_panrg+;kFT-3b68l!$Wyr zdue^05bNv@vkfB!&~P$FMw4RVpRhbfeet{DRNzho=leuq{BQxBSuUrb2ZkH5>Q+^d z9lZv{cYB}{sll3wjQ(U1Ig>GjGimt+G$KUN1HotxQ*Hu)wdQpEFYfuG}k z`Lb1DP-==Qy{MhN|HRzuLrB)AoB&vvdV2rkgwW%HTguYM#U-+|4t^&RQi03~3ER|8 z3`NG+mkWFpk-H}25;3coGYGdB-BWv-q1U*>H8o;7Qd3~4Ys}K|Dx3vtD<8-`=F3emu=-G%J^w2j8 z!DJ3hu6%KhyuS3ws$OxWowu2SYb2>4TL`S96z%Zz=(hVrG*o?QptKAM86<73;bZS6(w7cH?^!8e`H zj>*R`;~^X3PH%X|;g9P$`cIBRzqOx439@NqiXzBw6z^R8TJ)`IVb&5QQ*j5UCfzZ} z4*GD=9Wa<1>7j4e5B1PMM;4{*smu(Z)-usRhlTcIbJh$_bD^=BdMOS&gU=P-xAtro znKED?-W!x6?%t~gbGV&S@08&sDAk*k*YVuQg1LQoaok+MrMMvOLdsgf`>$nd&rzb1 z6S-OP-0^+c=kS|XAk8!Z#`8>L!}XB>p-wd@-Db{!+q0-plqYvKo(EqRl@LWxACRQ7W4#;2Q{07ABhvKn3O^iitv?7 zYCu@JvaPdWZ#l0=;OLDS8N{;`io|RAX}<%4*2*?hKbpC&exHy(T=-ow0!n{?Hsx>M ze|4-8I5IW{Yn;LAplqCX_jH1k@t-{d9ZLk4`$-wxJuFY#znm6}b4pR1N*P#|=eR6Y zIUAu_@C&367j-CeX$1?6k5Z!`+cmHlsF8_>?tAxFGCp1>R@rZ8LT*WC6dJioIT%A? zNT(=r20>@W!O$BpXhS_WChBcy!pO@DQT8HtJMt`F7}fe$^-g+Lu9-pi>FmIy*`!rR zpI#~SXBv#;pw~K)6wHT9>A$zIN>*tJw5Ra9AV__z0~NA@A{3tL+mo|2g6!f&pLe1b z#TS|X!e56rk}fAoDQUr}A$$o@+yR7X;W!R|U4Psd7HE92$wkigFh_l?D}J>XyLK#l zx8fVl%P%Xfq>=NgERgO~_)&TrlBJ3-Bvpju5XT>x-%l(voo9EJqnc-cTAO2=M)rPQ z(#dt4T2DsgIv5OjF<1!-!d;FELcXK-`dr)lcEi zzkrH8jfD}oL_Su0yD~7x*zh4b%sf!@nkQqgpl(#}rw97S7n|JEa$rsQM%$LP!%k1Q zOti_J0m_RFn(_SSCTaOdAz`{EE`3G5DM50rYd`P=);QiTYBx_L@bcod z{T#Xoe)Fb+At{2|H)S#>%!gBpoLO0jt`|ER`>prWNu^0dK@CM@!ePtg`^k)8m8xDN zfaZmCp{Du7OWYyQ`bDvf7QglgJQ7&bEgFu9UBY?sw{(Pnh+2(=ZE{u$OD!Q{xsLN1 z@z@JRksfepx)~s)ASfc~Pm=O>M&wQ&Ov`+2iIduaoKwOV(t~z+Ju;}l5IQfG1->rY zCtPV+ZUbN{e?*?4N?9q0&g~Pru(V4j9GXgvCz{x;J{JX=MNe?!ic-5(1j~ z$`j$`rJ~H|;&hWb@2cYX`%9Kf)0>jo$$pJ^U^9#(< z@cx54QHs^sE6c${#~UE4v3DqwXR}|bvQ{kZQvM-thr|t610mMJmzrPvC%Xv`6YmCL zKE}-50@gHt9xhH7ZFGNKq{#~s$8+9Q59e#iSyy<$pb?g>x9^`r3xtId1I{FurKp!6!3n_g~Qwk1qIzRm)=yMwL)+I8hs4%V++haUG zv-i@~a)r`aSI#^~N4Ap=Zot53y!OV<=kQ;NuJb{2<5;e)c~dvRWq$IrPkEdgwx^V* z6EFtNC_!T%L#|A8^HTCyJ(XtX>q{F0^Un@>YiL5qO8}lQFv}n@VU91PiJ8U+M+nQoY9JXsPNlszwU=7CzS23g0`7V%99&?(< z&-;v~NtQb}neTySG7lRY+|g>hkfO7lfq79d<3}umY|zT^z+yUOcqj@f2yX{f2a{=F zkwyCyItFcsI+!+Ulnna^EyF%)GV(bqyv9Y35`(kU<2n^AJ=WM!8KWGfp-4PInhb8HSLiOp)aXUC8FOFgDFvf5EuNfjIxm81E1ixdBPx zT}&wHMIP^^X7e0-#vkY$T}a8|cJL(O2B+EhvV-0-V}t-%hK@^;)pS@kZ@Qi5)`7jR zy_=e~ZRWv%V?dMaz|!tLc8oSX z5iiD?nyxnSnWT96?S0}*LR!N=o-^t6*W^JOp5{FAFce=)clec^OrMBSiyI!kNAq|} zr)ASfRh_#D!zGbBIm)hju>=#cYi#Gq>B_raaAuJb<()ko49`s=!OrGah}MGOQPdWvNmy~ z8aa21FL;mnB&6tLo^VOza}`lL2_N29nljH<;c^IW6%+X`GYc;5ogYbLyXuWGSA8i| zpGT$>%tz39i(04FqX1n^_F5Y>oCmcg!oxZ6ldNhryvOqsUy;dPE$~Zx&nZN)!&*Gf zB0p1f^6{hi!O`waws`T2EN-jo6KeIZr+7grIXnsj6MTuHJ`@P*i_X@U9iEx^Jp08b z`xoC6gWeT=?Oyd#g`6Qqo=yqxBS|c3XJ)WGa#K1t0!6 zWM7d@sW7gHZhv7!o~7F*yWji;Gu%9%H~hduBExAxqxOE1vmLcG{}Qymcp?Na}C zwVM&k>aV2HT=yy#aEBFhxmnFHCa@7u1Ilx-ABa_xV;h3$z&k=Un_AIPG}!c%=Qs_S z;N_5f_Z83BDP@XqHB8$S#*?cB+)Xdmg?|E)=`X-{a{7w+*8Mh&-u@M3#tXG|8Qg)- zr+L@haVlag`g#1to>imix9#7*iyMg_90o4mrEqF4qPUOxBOdJ1YUtg q{+8YG_bZkf +{%- endif %} +- Deployment environment: `{{target_environment}}` +{%- if zenml_server_url!='' %} +- Remote ZenML Server URL: `{{zenml_server_url}}` +{%- endif %} + +Settings of your project are: +- Dataset: `{{dataset}}` +- Model: `{{model}}` +- Every trained model will be promoted to `{{target_environment}}` +{%- if notify_on_failures and notify_on_successes %} +- Notifications about failures and successes enabled +{%- elif notify_on_failures %} +- Notifications about failures enabled +{%- elif notify_on_successes %} +- Notifications about success enabled +{%- else %} +- All notifications disabled +{%- endif %} + +## 👋 Introduction + +Welcome to your newly generated "{{project_name}}" project! This is +a great way to get hands-on with ZenML using production-like template. +The project contains a collection of standard and custom ZenML steps, +pipelines and other artifacts and useful resources that can serve as a +solid starting point for your smooth journey with ZenML. + +What to do first? You can start by giving the the project a quick run. The +project is ready to be used and can run as-is without any further code +changes! You can try it right away by installing ZenML, the needed +ZenML integration and then calling the CLI included in the project. We also +recommend that you start the ZenML UI locally to get a better sense of what +is going on under the hood: + +```bash +# Set up a Python virtual environment, if you haven't already +python3 -m venv .venv +source .venv/bin/activate +# Install requirements & integrations +make setup +# Optionally, provision default local stack +make install-stack +# Start the ZenML UI locally (recommended, but optional); +# the default username is "admin" with an empty password +zenml up +# Run the pipeline included in the project +python run.py +``` + +When the pipelines are done running, you can check out the results in the ZenML +UI by following the link printed in the terminal (or you can go straight to +the [ZenML UI pipelines run page](http://127.0.0.1:8237/workspaces/default/all-runs?page=1). + +Next, you should: + +* look at the CLI help to see what you can do with the project: +```bash +python run.py --help +``` diff --git a/template/artifacts/__init__.py b/template/artifacts/__init__.py new file mode 100644 index 0000000..4bc11e5 --- /dev/null +++ b/template/artifacts/__init__.py @@ -0,0 +1 @@ +# {% include 'template/license_header' %} diff --git a/template/artifacts/materializer.py b/template/artifacts/materializer.py new file mode 100644 index 0000000..40c89a3 --- /dev/null +++ b/template/artifacts/materializer.py @@ -0,0 +1,89 @@ +# {% include 'template/license_header' %} + + +import json +import os +from typing import Type + +from zenml.enums import ArtifactType +from zenml.io import fileio +from zenml.materializers.base_materializer import BaseMaterializer + +from artifacts.model_metadata import ModelMetadata + + +class ModelMetadataMaterializer(BaseMaterializer): + ASSOCIATED_TYPES = (ModelMetadata,) + ASSOCIATED_ARTIFACT_TYPE = ArtifactType.STATISTICS + + def load(self, data_type: Type[ModelMetadata]) -> ModelMetadata: + """Read from artifact store. + + Args: + data_type: What type the artifact data should be loaded as. + + Raises: + ValueError: on deserialization issue + + Returns: + Read artifact. + """ + super().load(data_type) + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + import sklearn.ensemble + import sklearn.linear_model + import sklearn.tree + + modules = [sklearn.ensemble, sklearn.linear_model, sklearn.tree] + + with fileio.open(os.path.join(self.uri, "data.json"), "r") as f: + data_json = json.loads(f.read()) + class_name = data_json["model_class"] + cls = None + for module in modules: + if cls := getattr(module, class_name, None): + break + if cls is None: + raise ValueError( + f"Cannot deserialize `{class_name}` using {self.__class__.__name__}. " + f"Only classes from modules {[m.__name__ for m in modules]} " + "are supported" + ) + data = ModelMetadata(cls) + if "search_grid" in data_json: + data.search_grid = data_json["search_grid"] + if "params" in data_json: + data.params = data_json["params"] + if "metric" in data_json: + data.metric = data_json["metric"] + ### YOUR CODE ENDS HERE ### + + return data + + def save(self, data: ModelMetadata) -> None: + """Write to artifact store. + + Args: + data: The data of the artifact to save. + """ + super().save(data) + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Dump the model metadata directly into the artifact store as a JSON file + data_json = dict() + with fileio.open(os.path.join(self.uri, "data.json"), "w") as f: + data_json["model_class"] = data.model_class.__name__ + if data.search_grid: + data_json["search_grid"] = {} + for k, v in data.search_grid.items(): + if type(v) == range: + data_json["search_grid"][k] = list(v) + else: + data_json["search_grid"][k] = v + if data.params: + data_json["params"] = data.params + if data.metric: + data_json["metric"] = data.metric + f.write(json.dumps(data_json)) + ### YOUR CODE ENDS HERE ### diff --git a/template/artifacts/model_metadata.py b/template/artifacts/model_metadata.py new file mode 100644 index 0000000..d3e6ea6 --- /dev/null +++ b/template/artifacts/model_metadata.py @@ -0,0 +1,62 @@ +# {% include 'template/license_header' %} + + +from typing import Any, Dict + +from sklearn.base import ClassifierMixin + + +class ModelMetadata: + """A custom artifact that stores model metadata. + + A model metadata object gathers together information that is collected + about the model being trained in a training pipeline run. This data type + is used for one of the artifacts returned by the model evaluation step. + + This is an example of a *custom artifact data type*: a type returned by + one of the pipeline steps that isn't natively supported by the ZenML + framework. Custom artifact data types are a common occurrence in ZenML, + usually encountered in one of the following circumstances: + + - you use a third party library that is not covered as a ZenML integration + and you model one or more step artifacts from the data types provided by + this library (e.g. datasets, models, data validation profiles, model + evaluation results/reports etc.) + - you need to use one of your own data types as a step artifact and it is + not one of the basic Python artifact data types supported by the ZenML + framework (e.g. str, int, float, dictionaries, lists, etc.) + - you want to extend one of the artifact data types already natively + supported by ZenML (e.g. pandas.DataFrame or sklearn.ClassifierMixin) + to customize it with your own data and/or behavior. + + In all above cases, the ZenML framework lacks one very important piece of + information: it doesn't "know" how to convert the data into a format that + can be saved in the artifact store (e.g. on a filesystem or persistent + storage service like S3 or GCS). Saving and loading artifacts from the + artifact store is something called "materialization" in ZenML terms and + you need to provide this missing information in the form of a custom + materializer - a class that implements loading/saving artifacts from/to + the artifact store. Take a look at the `materializers` folder to see how a + custom materializer is implemented for this artifact data type. + + More information about custom step artifact data types and ZenML + materializers is available in the docs: + + https://docs.zenml.io/user-guide/advanced-guide/artifact-management/handle-custom-data-types + + """ + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + def __init__( + self, + model_class: ClassifierMixin, + search_grid: Dict[str, Any] = None, + params: Dict[str, Any] = None, + metric: float = None, + ) -> None: + self.model_class = model_class + self.search_grid = search_grid + self.params = params + self.metric = metric + + ### YOUR CODE ENDS HERE ### diff --git a/template/config.py b/template/config.py new file mode 100644 index 0000000..2d4a5bf --- /dev/null +++ b/template/config.py @@ -0,0 +1,80 @@ +# {% include 'template/license_header' %} + + +from artifacts.model_metadata import ModelMetadata +from pydantic import BaseConfig + +from zenml.config import DockerSettings +from utils.misc import ( + HFSentimentAnalysisDataset, + HFPretrainedModel, + HFPretrainedTokenizer, +) +from zenml.integrations.constants import ( +{%- if cloud_of_choice == 'AWS' %} + SKYPILOT_AWS, + AWS, + S3, +{%- endif %} +{%- if cloud_of_choice == 'GCP' %} + SKYPILOT_GCP, + GCP, +{%- endif %} + HUGGINGFACE, + PYTORCH, + MLFLOW, + SLACK, + +) +from zenml.model_registries.base_model_registry import ModelVersionStage + +PIPELINE_SETTINGS = dict( + docker=DockerSettings( + required_integrations=[ + {%- if cloud_of_choice == 'AWS' %} + SKYPILOT_AWS, + AWS, + S3, + {%- endif %} + {%- if cloud_of_choice == 'GCP' %} + SKYPILOT_GCP, + GCP, + {%- endif %} + HUGGINGFACE, + PYTORCH, + MLFLOW, + SLACK, + ], + ) +) + +DEFAULT_PIPELINE_EXTRAS = dict( + notify_on_success={{notify_on_successes}}, + notify_on_failure={{notify_on_failures}} +) + +class MetaConfig(BaseConfig): +{%- if dataset == 'imbd_reviews' %} + dataset = HFSentimentAnalysisDataset.imbd_reviews +{%- endif %} +{%- if dataset == 'airline_reviews' %} + dataset = HFSentimentAnalysisDataset.airline_reviews +{%- else %} + dataset = HFSentimentAnalysisDataset.financial_news +{%- endif %} +{%- if model == 'gpt2' %} + tokenizer = HFPretrainedTokenizer.gpt2 + model = HFPretrainedModel.gpt2 +{%- else %} + tokenizer = HFPretrainedTokenizer.bert + model = HFPretrainedModel.bert +{%- endif %} + pipeline_name_training = "{{product_name}}_training" + mlflow_model_name = "{{product_name}}_model" +{%- if target_environment == 'production' %} + target_env = ModelVersionStage.PRODUCTION +{%- else %} + target_env = ModelVersionStage.STAGING +{%- endif %} + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### \ No newline at end of file diff --git a/template/license b/template/license new file mode 100644 index 0000000..3697641 --- /dev/null +++ b/template/license @@ -0,0 +1,110 @@ +{% if open_source_license == 'mit' -%} +MIT License + +Copyright (c) {{ full_name }} {% now 'local', '%Y' %} + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +{% elif open_source_license == 'bsd' %} + +BSD License + +Copyright (c) {{ full_name }} {% now 'local', '%Y' %}. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, +INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED +OF THE POSSIBILITY OF SUCH DAMAGE. +{% elif open_source_license == 'isc' -%} +ISC License + +Copyright (c) {{ full_name }} {% now 'local', '%Y' %} + +Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +{% elif open_source_license == 'apache' -%} +Apache Software License 2.0 + +Copyright (c) {{ full_name }} {% now 'local', '%Y' %}. All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +{% elif open_source_license == 'gpl3' -%} +GNU GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + {{ project_short_description }} + Copyright (C) {{ full_name }} {% now 'local', '%Y' %} + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU GPL, see +. + + The GNU General Public License does not permit incorporating your program +into proprietary programs. If your program is a subroutine library, you +may consider it more useful to permit linking proprietary applications with +the library. If this is what you want to do, use the GNU Lesser General +Public License instead of this License. But first, please read +. +{% endif %} \ No newline at end of file diff --git a/template/license_header b/template/license_header new file mode 100644 index 0000000..cc653b6 --- /dev/null +++ b/template/license_header @@ -0,0 +1,2 @@ +{%- macro license() %}{% include 'template/license' %}{% endmacro -%} +{{ license() | replace('\n', '\n# ') }} \ No newline at end of file diff --git a/template/pipelines/__init__.py b/template/pipelines/__init__.py new file mode 100644 index 0000000..67a5b51 --- /dev/null +++ b/template/pipelines/__init__.py @@ -0,0 +1,4 @@ +# {% include 'template/license_header' %} + + +from .training import {{product_name}}_training diff --git a/template/pipelines/training.py b/template/pipelines/training.py new file mode 100644 index 0000000..3a2a135 --- /dev/null +++ b/template/pipelines/training.py @@ -0,0 +1,115 @@ +# {% include 'template/license_header' %} + + +from typing import List, Optional + +from config import DEFAULT_PIPELINE_EXTRAS, PIPELINE_SETTINGS, MetaConfig +from steps import ( + data_loader, + model_trainer, + notify_on_failure, + notify_on_success, + promote_latest, + promote_get_versions, +) +from zenml import pipeline +from zenml.integrations.mlflow.steps.mlflow_deployer import ( + mlflow_model_registry_deployer_step, +) +from zenml.integrations.mlflow.steps.mlflow_registry import mlflow_register_model_step +from zenml.logger import get_logger +from zenml.steps.external_artifact import ExternalArtifact + + +logger = get_logger(__name__) + + +@pipeline( + settings=PIPELINE_SETTINGS, + on_failure=notify_on_failure, + extra=DEFAULT_PIPELINE_EXTRAS, +) +def {{product_name}}_training( + #hf_dataset: HFSentimentAnalysisDataset, + #hf_tokenizer: HFPretrainedTokenizer, + #hf_pretrained_model: HFPretrainedModel, + lower_case: bool = True, + padding: Optional[str] = "max_length", + max_seq_length: Optional[int] = 128, + text_column: Optional[str] = "text", + label_column: Optional[str] = "label", + train_batch_size: Optional[int] = 16, + eval_batch_size: Optional[int] = 16, + epochs: Optional[int] = 3, + learning_rate: Optional[float] = 2e-5, + weight_decay: Optional[float] = 0.01, +): + """ + Model training pipeline. + + This is a pipeline that loads the data, processes it and splits + it into train and test sets, then search for best hyperparameters, + trains and evaluates a model. + + Args: + test_size: Size of holdout set for training 0.0..1.0 + drop_na: If `True` NA values will be removed from dataset + normalize: If `True` dataset will be normalized with MinMaxScaler + drop_columns: List of columns to drop from dataset + random_seed: Seed of random generator, + min_train_accuracy: Threshold to stop execution if train set accuracy is lower + min_test_accuracy: Threshold to stop execution if test set accuracy is lower + fail_on_accuracy_quality_gates: If `True` and `min_train_accuracy` or `min_test_accuracy` + are not met - execution will be interrupted early + + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Link all the steps together by calling them and passing the output + # of one step as the input of the next step. + ########## Tokenization stage ########## + dataset = data_loader( + hf_dataset=MetaConfig.dataset, + ) + tokenizer = tokenizer_loader( + hf_tokenizer=MetaConfig.tokenizer, + lower_case=lower_case + ) + tokenized_data = tokenization_step( + dataset=dataset, + tokenizer=tokenizer, + padding=padding, + max_seq_length=max_seq_length, + text_column=text_column, + label_column=label_column + ) + + + ########## Training stage ########## + model = model_trainer( + tokenized_dataset=tokenized_data, + hf_pretrained_model=MetaConfig.model, + tokenizer=tokenizer, + train_batch_size=train_batch_size, + eval_batch_size=eval_batch_size, + num_epochs=epochs, + learning_rate=learning_rate, + weight_decay=weight_decay, + ) + + mlflow_register_model_step( + model, + name=MetaConfig.mlflow_model_name, + ) + + ########## Promotion stage ########## + latest_version, current_version = promote_get_versions( + after=["mlflow_register_model_step"], + ) + promote_latest( + latest_version=latest_version, + current_version=current_version, + ) + last_step_name = "promote_latest" + + notify_on_success(after=[last_step_name]) + ### YOUR CODE ENDS HERE ### diff --git a/template/requirements.txt b/template/requirements.txt new file mode 100644 index 0000000..75c1620 --- /dev/null +++ b/template/requirements.txt @@ -0,0 +1 @@ +zenml[server] diff --git a/template/run.py b/template/run.py new file mode 100644 index 0000000..6b8e3b2 --- /dev/null +++ b/template/run.py @@ -0,0 +1,169 @@ +# {% include 'template/license_header' %} + + +from zenml.steps.external_artifact import ExternalArtifact +from zenml.logger import get_logger +from pipelines import {{product_name}}_training +from config import MetaConfig +import click +from typing import Optional +from datetime import datetime as dt + +logger = get_logger(__name__) + + +@click.command( + help=""" +{{ project_name }} CLI v{{ version }}. + +Run the {{ project_name }} model training pipeline with various +options. + +Examples: + + + \b + # Run the pipeline with default options + python run.py + + \b + # Run the pipeline without cache + python run.py --no-cache + + \b + # Run the pipeline without NA drop and normalization, + # but dropping columns [A,B,C] and keeping 10% of dataset + # as test set. + python run.py --num-epochs 3 --train-batch-size 8 --eval-batch-size 8 + + \b + # Run the pipeline with Quality Gate for accuracy set at 90% for train set + # and 85% for test set. If any of accuracies will be lower - pipeline will fail. + python run.py --min-train-accuracy 0.9 --min-test-accuracy 0.85 --fail-on-accuracy-quality-gates + + +""" +) +@click.option( + "--no-cache", + is_flag=True, + default=False, + help="Disable caching for the pipeline run.", +) +@click.option( + "--num-epochs", + default=5, + type=click.INT, + help="Number of epochs to train the model for.", +) +@click.option( + "--seed", + default=42, + type=click.INT, + help="Seed for the random number generator.", +) +@click.option( + "--train-batch-size", + default=16, + type=click.INT, + help="Batch size for training the model.", +) +@click.option( + "--eval-batch-size", + default=16, + type=click.INT, + help="Batch size for evaluating the model.", +) +@click.option( + "--learning-rate", + default=2e-5, + type=click.FLOAT, + help="Learning rate for training the model.", +) +@click.option( + "--weight-decay", + default=0.01, + type=click.FLOAT, + help="Weight decay for training the model.", +) +@click.option( + "--fail-on-accuracy-quality-gates", + is_flag=True, + default=False, + help="Whether to fail the pipeline run if the model evaluation step " + "finds that the model is not accurate enough.", +) +def main( + no_cache: bool = False, + seed: int = 42, + num_epochs: int = 5, + train_batch_size: int = 16, + eval_batch_size: int = 16, + learning_rate: float = 2e-5, + weight_decay: float = 0.01, + min_train_accuracy: float = 0.8, + min_test_accuracy: float = 0.8, + fail_on_accuracy_quality_gates: bool = False, +): + """Main entry point for the pipeline execution. + + This entrypoint is where everything comes together: + + * configuring pipeline with the required parameters + (some of which may come from command line arguments) + * launching the pipeline + + Args: + no_cache: If `True` cache will be disabled. + test_size: Percentage of records from the training dataset to go into the test dataset. + min_train_accuracy: Minimum acceptable accuracy on the train set. + min_test_accuracy: Minimum acceptable accuracy on the test set. + fail_on_accuracy_quality_gates: If `True` and any of minimal accuracy + thresholds are violated - the pipeline will fail. If `False` thresholds will + not affect the pipeline. + """ + + # Run a pipeline with the required parameters. This executes + # all steps in the pipeline in the correct order using the orchestrator + # stack component that is configured in your active ZenML stack. + pipeline_args = {} + if no_cache: + pipeline_args["enable_cache"] = False + + # Execute Training Pipeline + run_args_train = { + "seed": seed, + "num_epochs": num_epochs, + "train_batch_size": train_batch_size, + "eval_batch_size": eval_batch_size, + "learning_rate": learning_rate, + "weight_decay": weight_decay, + "fail_on_accuracy_quality_gates": fail_on_accuracy_quality_gates, + } + + pipeline_args[ + "run_name" + ] = f"{MetaConfig.pipeline_name_training}_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + {{product_name}}_training.with_options(**pipeline_args)(**run_args_train) + logger.info("Training pipeline finished successfully!") + + # Execute Batch Inference Pipeline + #run_args_inference = {} + #pipeline_args[ + # "run_name" + #] = f"{MetaConfig.pipeline_name_batch_inference}_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + #{{product_name}}_batch_inference.with_options(**pipeline_args)(**run_args_inference) + + #artifact = ExternalArtifact( + # pipeline_name=MetaConfig.pipeline_name_batch_inference, + # artifact_name="predictions", + #) + #logger.info( + # "Batch inference pipeline finished successfully! " + # "You can find predictions in Artifact Store using ID: " + # f"`{str(artifact.upload_if_necessary())}`." + #) + + +if __name__ == "__main__": + main() diff --git a/template/steps/__init__.py b/template/steps/__init__.py new file mode 100644 index 0000000..3f178a5 --- /dev/null +++ b/template/steps/__init__.py @@ -0,0 +1,19 @@ +# {% include 'template/license_header' %} + + +from .alerts import notify_on_failure, notify_on_success +from .dataset_loader import ( + data_loader, +) +from .tokenizer_loader import ( + tokenizer_loader, +) +from .tokenzation import ( + tokenization_step, +) +from .inference import inference_get_current_version, inference_predict +from .promotion import ( + promote_latest, + promote_get_versions, +) +from .training import model_trainer diff --git a/template/steps/alerts/__init__.py b/template/steps/alerts/__init__.py new file mode 100644 index 0000000..31351d3 --- /dev/null +++ b/template/steps/alerts/__init__.py @@ -0,0 +1,4 @@ +# {% include 'template/license_header' %} + + +from .notify_on import notify_on_failure, notify_on_success diff --git a/template/steps/alerts/notify_on.py b/template/steps/alerts/notify_on.py new file mode 100644 index 0000000..ae183a9 --- /dev/null +++ b/template/steps/alerts/notify_on.py @@ -0,0 +1,42 @@ +# {% include 'template/license_header' %} + + +from zenml import get_step_context, step +from zenml.client import Client +from zenml.utils.dashboard_utils import get_run_url + +alerter = Client().active_stack.alerter + + +def build_message(status: str) -> str: + """Builds a message to post. + + Args: + status: Status to be set in text. + + Returns: + str: Prepared message. + """ + step_context = get_step_context() + run_url = get_run_url(step_context.pipeline_run) + + return ( + f"Pipeline `{step_context.pipeline.name}` [{str(step_context.pipeline.id)}] {status}!\n" + f"Run `{step_context.pipeline_run.name}` [{str(step_context.pipeline_run.id)}]\n" + f"URL: {run_url}" + ) + + +def notify_on_failure() -> None: + """Notifies user on step failure. Used in Hook.""" + step_context = get_step_context() + if alerter and step_context.pipeline_run.config.extra["notify_on_failure"]: + alerter.post(message=build_message(status="failed")) + + +@step(enable_cache=False) +def notify_on_success() -> None: + """Notifies user on pipeline success.""" + step_context = get_step_context() + if alerter and step_context.pipeline_run.config.extra["notify_on_success"]: + alerter.post(message=build_message(status="succeeded")) diff --git a/template/steps/dataset_loader/__init__.py b/template/steps/dataset_loader/__init__.py new file mode 100644 index 0000000..e51200a --- /dev/null +++ b/template/steps/dataset_loader/__init__.py @@ -0,0 +1,4 @@ +# {% include 'template/license_header' %} + + +from .data_loader import data_loader diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py new file mode 100644 index 0000000..baf1bd4 --- /dev/null +++ b/template/steps/dataset_loader/data_loader.py @@ -0,0 +1,61 @@ +# {% include 'template/license_header' %} + + +from typing_extensions import Annotated + +from datasets import load_dataset, DatasetDict + +from zenml import step +from zenml.logger import get_logger + +from config import HFSentimentAnalysisDataset + +logger = get_logger(__name__) + + +@step +def data_loader( + hf_dataset: HFSentimentAnalysisDataset, +) -> Annotated[DatasetDict, "dataset"]: + """Data loader step. + + This is an example of a data loader step that is usually the first step + in your pipeline. It reads data from an external source like a file, + database or 3rd party library, then formats it and returns it as a step + output artifact. + + This step is parameterized using the `DataLoaderStepParameters` class, which + allows you to configure the step independently of the step code, before + running it in a pipeline. In this example, the step can be configured to + load different built-in scikit-learn datasets. See the documentation for + more information: + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Data loader steps should have caching disabled if they are not deterministic + (i.e. if they data they load from the external source can be different when + they are subsequently called, even if the step code and parameter values + don't change). + + Args: + params: Parameters for the data loader step. + + Returns: + The loaded dataset artifact. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Load the dataset indicated in the step parameters and format it as a + # pandas DataFrame + logger.info(f"Loaded dataset {hf_dataset.value}") + if ( + hf_dataset == HFSentimentAnalysisDataset.financial_news + or hf_dataset != HFSentimentAnalysisDataset.imbd_reviews + and hf_dataset == HFSentimentAnalysisDataset.airline_reviews + ): + dataset = load_dataset(hf_dataset.value) + elif hf_dataset == HFSentimentAnalysisDataset.imbd_reviews: + dataset = load_dataset(hf_dataset.value, split='train') + logger.info(dataset) + logger.info("Sample Example :", dataset["train"][1]) + ### YOUR CODE ENDS HERE ### + return dataset \ No newline at end of file diff --git a/template/steps/inference/__init__.py b/template/steps/inference/__init__.py new file mode 100644 index 0000000..fc63455 --- /dev/null +++ b/template/steps/inference/__init__.py @@ -0,0 +1,5 @@ +# {% include 'template/license_header' %} + + +from .inference_get_current_version import inference_get_current_version +from .inference_predict import inference_predict diff --git a/template/steps/inference/inference_get_current_version.py b/template/steps/inference/inference_get_current_version.py new file mode 100644 index 0000000..0454d87 --- /dev/null +++ b/template/steps/inference/inference_get_current_version.py @@ -0,0 +1,34 @@ +# {% include 'template/license_header' %} + + +from typing_extensions import Annotated + +from config import MetaConfig +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger + +logger = get_logger(__name__) + +model_registry = Client().active_stack.model_registry + + +@step +def inference_get_current_version() -> Annotated[str, "model_version"]: + """Get currently tagged model version for deployment. + + Returns: + The model version of currently tagged model in Registry. + """ + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + + current_version = model_registry.list_model_versions( + name=MetaConfig.mlflow_model_name, + stage=MetaConfig.target_env, + )[0].version + logger.info( + f"Current model version in `{MetaConfig.target_env.value}` is `{current_version}`" + ) + + return current_version diff --git a/template/steps/inference/inference_predict.py b/template/steps/inference/inference_predict.py new file mode 100644 index 0000000..3406583 --- /dev/null +++ b/template/steps/inference/inference_predict.py @@ -0,0 +1,43 @@ +# {% include 'template/license_header' %} + + +from typing_extensions import Annotated + +import pandas as pd +from zenml import step +from zenml.integrations.mlflow.model_deployers.mlflow_model_deployer import ( + MLFlowDeploymentService, +) + + +@step +def inference_predict( + deployment_service: MLFlowDeploymentService, + dataset_inf: pd.DataFrame, +) -> Annotated[pd.Series, "predictions"]: + """Predictions step. + + This is an example of a predictions step that takes the data in and returns + predicted values. + + This step is parameterized, which allows you to configure the step + independently of the step code, before running it in a pipeline. + In this example, the step can be configured to use different input data + and model version in registry. See the documentation for more information: + + https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines + + Args: + deployment_service: Deployed model service. + dataset_inf: The inference dataset. + + Returns: + The processed dataframe: dataset_inf. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + predictions = deployment_service.predict(request=dataset_inf) + predictions = pd.Series(predictions, name="predicted") + deployment_service.deprovision(force=True) + ### YOUR CODE ENDS HERE ### + + return predictions diff --git a/template/steps/promotion/__init__.py b/template/steps/promotion/__init__.py new file mode 100644 index 0000000..aca59cf --- /dev/null +++ b/template/steps/promotion/__init__.py @@ -0,0 +1,5 @@ +# {% include 'template/license_header' %} + + +from .promote_latest import promote_latest +from .promote_get_versions import promote_get_versions diff --git a/template/steps/promotion/promote_get_versions.py b/template/steps/promotion/promote_get_versions.py new file mode 100644 index 0000000..4366a37 --- /dev/null +++ b/template/steps/promotion/promote_get_versions.py @@ -0,0 +1,50 @@ +# {% include 'template/license_header' %} + + +from typing import Tuple +from typing_extensions import Annotated + +from config import MetaConfig +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger + +logger = get_logger(__name__) + +model_registry = Client().active_stack.model_registry + + +@step +def promote_get_versions() -> ( + Tuple[Annotated[str, "latest_version"], Annotated[str, "current_version"]] +): + """Step to get latest and currently tagged model version from Model Registry. + + This is an example of a model version extraction step. It will retrieve 2 model + versions from Model Registry: latest and currently promoted to target + environment (Production, Staging, etc). + + Returns: + The model versions: latest and current. If not current version - returns same + for both. + """ + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + none_versions = model_registry.list_model_versions( + name=MetaConfig.mlflow_model_name, + stage=None, + ) + latest_versions = none_versions[0].version + logger.info(f"Latest model version is {latest_versions}") + + target_versions = model_registry.list_model_versions( + name=MetaConfig.mlflow_model_name, + stage=MetaConfig.target_env, + ) + current_version = latest_versions + if target_versions: + current_version = target_versions[0].version + logger.info(f"Currently promoted model version is {current_version}") + else: + logger.info("No currently promoted model version found.") + return current_version, current_version diff --git a/template/steps/promotion/promote_latest.py b/template/steps/promotion/promote_latest.py new file mode 100644 index 0000000..6b940fe --- /dev/null +++ b/template/steps/promotion/promote_latest.py @@ -0,0 +1,49 @@ +# {% include 'template/license_header' %} + + +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger +from zenml.model_registries.base_model_registry import ModelVersionStage + +from config import MetaConfig + +logger = get_logger(__name__) + +model_registry = Client().active_stack.model_registry + + +@step +def promote_latest(latest_version:str, current_version:str): + """Promote latest trained model. + + This is an example of a model promotion step, which promotes the + latest trained model to the current version. + + Args: + latest_version: Recently trained model version. + current_version: Current model version, if present. + + """ + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + logger.info(f"Promoting latest model version `{latest_version}`") + if latest_version != current_version: + model_registry.update_model_version( + name=MetaConfig.mlflow_model_name, + version=current_version, + stage=ModelVersionStage.ARCHIVED, + metadata={}, + ) + model_registry.update_model_version( + name=MetaConfig.mlflow_model_name, + version=latest_version, + stage=MetaConfig.target_env, + metadata={}, + ) + promoted_version = latest_version + + logger.info( + f"Current model version in `{MetaConfig.target_env.value}` is `{promoted_version}`" + ) + ### YOUR CODE ENDS HERE ### diff --git a/template/steps/tokenizer_loader/__init__.py b/template/steps/tokenizer_loader/__init__.py new file mode 100644 index 0000000..b750592 --- /dev/null +++ b/template/steps/tokenizer_loader/__init__.py @@ -0,0 +1,4 @@ +# {% include 'template/license_header' %} + + +from .tokenizer_loader import tokenizer_loader diff --git a/template/steps/tokenizer_loader/tokenizer_loader.py b/template/steps/tokenizer_loader/tokenizer_loader.py new file mode 100644 index 0000000..0265690 --- /dev/null +++ b/template/steps/tokenizer_loader/tokenizer_loader.py @@ -0,0 +1,53 @@ +# {% include 'template/license_header' %} + + +from typing_extensions import Annotated +from transformers import BertTokenizer, GPT2Tokenizer, PreTrainedTokenizerBase + +from zenml.enums import StrEnum +from zenml import step +from zenml.logger import get_logger + +logger = get_logger(__name__) + +class HFPretrainedTokenizer(StrEnum): + """HuggingFace Sentiment Analysis datasets.""" + bert = "bert-base-uncased" + gpt2 = "gpt2" + +@step +def tokenizer_loader( + hf_tokenizer: HFPretrainedTokenizer, + lower_case: bool, +) -> Annotated[PreTrainedTokenizerBase, "tokenzer"]: + """Tokenizer loader step. + + This is an example of a data processor step that prepares the data so that + it is suitable for model training. It takes in a dataset as an input step + artifact and performs any necessary preprocessing steps like cleaning, + feature engineering, feature selection, etc. It then returns the processed + dataset as a step output artifact. + + This step is parameterized using the `DataProcessorStepParameters` class, + which allows you to configure the step independently of the step code, + before running it in a pipeline. In this example, the step can be configured + to perform or skip different preprocessing steps (e.g. dropping rows with + missing values, dropping columns, normalizing the data, etc.). See the + documentation for more information: + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Args: + params: Parameters for the data processor step. + + Returns: + The processed dataset artifact. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + if hf_tokenizer == HFPretrainedTokenizer.bert: + tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=lower_case) + elif hf_tokenizer == HFPretrainedTokenizer.gpt2: + tokenizer = GPT2Tokenizer.from_pretrained("gpt2", bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>') + ### YOUR CODE ENDS HERE ### + + return tokenizer \ No newline at end of file diff --git a/template/steps/tokenzation/__init__.py b/template/steps/tokenzation/__init__.py new file mode 100644 index 0000000..9bb6b25 --- /dev/null +++ b/template/steps/tokenzation/__init__.py @@ -0,0 +1,4 @@ +# {% include 'template/license_header' %} + + +from .tokenization import tokenization_step diff --git a/template/steps/tokenzation/tokenization.py b/template/steps/tokenzation/tokenization.py new file mode 100644 index 0000000..629d7db --- /dev/null +++ b/template/steps/tokenzation/tokenization.py @@ -0,0 +1,71 @@ +# {% include 'template/license_header' %} + + +from typing_extensions import Annotated +from transformers import PreTrainedTokenizerBase + +from datasets import DatasetDict +from zenml import step +from zenml.logger import get_logger + +from utils.misc import find_max_length + +logger = get_logger(__name__) + + +@step +def tokenization_step( + padding: str, + max_seq_length: int, + text_column: str, + label_column: str, + tokenizer: PreTrainedTokenizerBase, + dataset: DatasetDict, +) -> Annotated[DatasetDict, "tokenized_data"]: + """Data splitter step. + + This is an example of a data splitter step that splits the dataset into + training and dev subsets to be used for model training and evaluation. It + takes in a dataset as an step input artifact and returns the training and + dev subsets as two separate step output artifacts. + + Data splitter steps should have a deterministic behavior, i.e. they should + use a fixed random seed and always return the same split when called with + the same input dataset. This is to ensure reproducibility of your pipeline + runs. + + This step is parameterized using the `DataSplitterStepParameters` class, + which allows you to configure the step independently of the step code, + before running it in a pipeline. In this example, the step can be configured + to use a different random seed, change the split ratio, or control whether + to shuffle or stratify the split. See the documentation for more + information: + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Args: + params: Parameters for the data splitter step. + dataset: The dataset to split. + + Returns: + The resulting training and dev subsets. + """ + train_max_length = find_max_length(dataset["train"]["text"]) + val_max_length = find_max_length(dataset["validation"]["text"]) + max_length = train_max_length if train_max_length >= val_max_length else val_max_length + logger.info(f"max length for the given dataset is:{max_length}") + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + def preprocess_function(examples): + result = tokenizer( + examples[text_column], + #padding="max_length", + truncation=True, + max_length=max_length or max_seq_length + ) + result["label"] = examples[label_column] + return result + tokenized_datasets = dataset.map(preprocess_function, batched=True) + #tokenized_datasets = tokenized_datasets.remove_columns(["text"]) + #tokenized_datasets = tokenized_datasets.rename_column("label", "labels") + #tokenized_datasets.set_format("torch") + return tokenized_datasets \ No newline at end of file diff --git a/template/steps/training/__init__.py b/template/steps/training/__init__.py new file mode 100644 index 0000000..16dc6ed --- /dev/null +++ b/template/steps/training/__init__.py @@ -0,0 +1,3 @@ +# {% include 'template/license_header' %} + +from .model_trainer import model_trainer diff --git a/template/steps/training/model_trainer.py b/template/steps/training/model_trainer.py new file mode 100644 index 0000000..778adc1 --- /dev/null +++ b/template/steps/training/model_trainer.py @@ -0,0 +1,123 @@ +from typing import Tuple, Optional + +from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score +from transformers import TrainingArguments, Trainer, PreTrainedModel, DataCollatorWithPadding +from transformers import BertForSequenceClassification, GPT2ForSequenceClassification +from datasets import DatasetDict +from transformers import PreTrainedTokenizerBase +import mlflow + +from zenml import step +from zenml.enums import StrEnum +from zenml.client import Client + +from utils.misc import compute_metrics + +experiment_tracker = Client().active_stack.experiment_tracker + +class HFPretrainedModel(StrEnum): + """HuggingFace Sentiment Analysis Model.""" + bert = "bert-base-uncased" + gpt2 = "gpt2" + + +@step(experiment_tracker=experiment_tracker.name) +def model_trainer( + hf_pretrained_model: HFPretrainedModel, + tokenized_dataset: DatasetDict, + tokenizer: PreTrainedTokenizerBase, + num_labels: Optional[int] = 3, + train_batch_size: Optional[int] = 16, + num_epochs: Optional[int] = 3, + seed: Optional[int] = 42, + learning_rate: Optional[float] = 2e-5, + load_best_model_at_end: Optional[bool] = True, + eval_batch_size: Optional[int] = 16, + weight_decay: Optional[float] = 0.01, +) -> PreTrainedModel: + """Configure and train a model on the training dataset. + + This is an example of a model training step that takes in a dataset artifact + previously loaded and pre-processed by other steps in your pipeline, then + configures and trains a model on it. The model is then returned as a step + output artifact. + + Model training steps should have caching disabled if they are not + deterministic (i.e. if the model training involve some random processes + like initializing weights or shuffling data that are not controlled by + setting a fixed random seed). This example step ensures the outcome is + deterministic by initializing the model with a fixed random seed. + + This step is parameterized using the `ModelTrainerStepParameters` class, + which allows you to configure the step independently of the step code, +{%- if configurable_model %} + before running it in a pipeline. In this example, the step can be configured + to use a different model, change the random seed, or pass different + hyperparameters to the model constructor. See the documentation for more + information: +{%- else %} + before running it in a pipeline. In this example, the step can be configured + to change the random seed, or pass different hyperparameters to the model + constructor. See the documentation for more information: +{%- endif %} + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Args: + params: The parameters for the model trainer step. + train_set: The training data set artifact. + + Returns: + The trained model artifact. + """ + mlflow.transformers.autolog() + data_collator = DataCollatorWithPadding(tokenizer=tokenizer) + if hf_pretrained_model == HFPretrainedModel.bert: + model = BertForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=num_labels) + training_args = TrainingArguments( + output_dir="zenml_artifact", + learning_rate=learning_rate, + per_device_train_batch_size=train_batch_size, + per_device_eval_batch_size=eval_batch_size, + num_train_epochs=num_epochs, + weight_decay=weight_decay, + save_strategy="epoch", + load_best_model_at_end=load_best_model_at_end, + ) + trainer = Trainer( + model=model, + args=training_args, + train_dataset=tokenized_dataset["train"], + eval_dataset=tokenized_dataset["validation"], + compute_metrics=compute_metrics, + data_collator=data_collator, + ) + elif hf_pretrained_model == HFPretrainedModel.gpt2: + model = GPT2ForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=3) + model.resize_token_embeddings(len(tokenizer)) + training_args = TrainingArguments( + output_dir="zenml_artifact", + learning_rate=learning_rate, + per_device_train_batch_size=train_batch_size, + per_device_eval_batch_size=eval_batch_size, + num_train_epochs=num_epochs, + weight_decay=weight_decay, + save_strategy="epoch", + load_best_model_at_end=load_best_model_at_end, + ) + trainer = Trainer( + model=model, + args=training_args, + train_dataset=tokenized_dataset["train"], + eval_dataset=tokenized_dataset["validation"], + compute_metrics=compute_metrics, + data_collator=data_collator, + ) + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Initialize the model with the hyperparameters indicated in the step + # parameters and train it on the training set. + + ### YOUR CODE ENDS HERE ### + trainer.train() + trainer.evaluate() + return model \ No newline at end of file diff --git a/template/utils/misc.py b/template/utils/misc.py new file mode 100644 index 0000000..54872b0 --- /dev/null +++ b/template/utils/misc.py @@ -0,0 +1,39 @@ +# {% include 'template/license_header' %} + +import numpy as np +from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score + +from zenml.enums import StrEnum + +def compute_metrics(p): + pred, labels = p + pred = np.argmax(pred, axis=1) + + accuracy = accuracy_score(y_true=labels, y_pred=pred) + recall = recall_score(y_true=labels, y_pred=pred, average='micro') + precision = precision_score(y_true=labels, y_pred=pred, average='micro') + f1 = f1_score(y_true=labels, y_pred=pred, average='micro') + + return {"accuracy": accuracy, "precision": precision, "recall": recall, "f1": f1} + + + +def find_max_length(dataset): + return len(max(dataset, key=lambda x: len(x.split())).split()) + + +class HFSentimentAnalysisDataset(StrEnum): + """HuggingFace Sentiment Analysis datasets.""" + financial_news = "zeroshot/twitter-financial-news-sentiment" + imbd_reviews = "mushroomsolutions/imdb_sentiment_3000_Test" + airline_reviews = "mattbit/tweet-sentiment-airlines" + +class HFPretrainedModel(StrEnum): + """HuggingFace Sentiment Analysis Model.""" + bert = "bert-base-uncased" + gpt2 = "gpt2" + +class HFPretrainedTokenizer(StrEnum): + """HuggingFace Sentiment Analysis datasets.""" + bert = "bert-base-uncased" + gpt2 = "gpt2" \ No newline at end of file diff --git a/template/{% if open_source_license %}LICENSE{% endif %} b/template/{% if open_source_license %}LICENSE{% endif %} new file mode 100644 index 0000000..3de332b --- /dev/null +++ b/template/{% if open_source_license %}LICENSE{% endif %} @@ -0,0 +1 @@ +{% include 'template/license' %} \ No newline at end of file diff --git a/template/{{_copier_conf.answers_file}} b/template/{{_copier_conf.answers_file}} new file mode 100644 index 0000000..ea97bd4 --- /dev/null +++ b/template/{{_copier_conf.answers_file}} @@ -0,0 +1,2 @@ +# Changes here will be overwritten by Copier +{{ _copier_answers|to_nice_yaml -}} \ No newline at end of file diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..17859ea --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,6 @@ +autopep8 +pytest +pytest-randomly +ruff +black +isort From 4d26bebe931195096ac3f69be35e0ec835dc0945 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich Date: Fri, 6 Oct 2023 09:26:45 +0100 Subject: [PATCH 02/18] fix dependencies and add custom_training --- copier.yml | 5 +- template/.assets/00_pipelines_composition.png | Bin 77711 -> 0 bytes template/Makefile | 12 +-- template/config.py | 11 ++- template/pipelines/training.py | 8 +- template/run.py | 35 -------- template/steps/dataset_loader/data_loader.py | 1 + ...ining %}prepare_data_loaders.py{% endif %} | 64 ++++++++++++++ template/steps/training/model_trainer.py | 11 ++- ...m_training %}full_evaluation.py{% endif %} | 80 ++++++++++++++++++ ...tom_training %}full_training.py{% endif %} | 79 +++++++++++++++++ 11 files changed, 250 insertions(+), 56 deletions(-) delete mode 100644 template/.assets/00_pipelines_composition.png create mode 100644 template/steps/dataset_loader/{% if custom_training %}prepare_data_loaders.py{% endif %} create mode 100644 template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} create mode 100644 template/steps/training/{% if custom_training %}full_training.py{% endif %} diff --git a/copier.yml b/copier.yml index be5ab4c..a02e5b1 100644 --- a/copier.yml +++ b/copier.yml @@ -92,7 +92,10 @@ zenml_server_url: type: str help: "The URL of the ZenML server [Optional]" default: "" - +custom_training: + type: bool + help: "Whether to use custom training or not" + default: False # CONFIGURATION ------------------------- _templates_suffix: "" diff --git a/template/.assets/00_pipelines_composition.png b/template/.assets/00_pipelines_composition.png deleted file mode 100644 index bd614886baa43641d1224b0dab9d9fcb6b75019c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 77711 zcmeFZcTiK^_ct0uMFl}cg+~yqfT9!yL8&5&NVOtDh=7lXNRtvu2v`8=C@M%-1f*l3 zC6LgIC?LHhL3&FTKPz zXA1-Z*{XX{`wIA?Kp-1DHf;n~P~Gk&;A^wfMN=0DgjbOL!v%?r7X}x(T(0P7L2{c! zC%_-;ZO$5;g+M+=@UB>HfN;qk($zkD&4X(i2YY7TlOR5m^2xok!nyI72h|e2|IpL? z$w%dwY%l#!4ON-{*2i_I)eZ!{@|t~79)1fcf9OWU=HZ$bclQhIKX~NPwyKN0f!m&> z7kJi_tctYuMJ~9?n$|mX;*c>*`?JJOIoIk*pHalbwxyGEG@w3aR52fQ<7*u`-i`^G zt$C*pHx(gm64e3a93MYHrA-`vLLi#bPnZ6>kbGDR@z;fx9sAY)x^OROoA_TBzGy!a z`0K*01Jyi#T{ymdkel;@-|I~jF32{HOZNi*4^7H3r>5TjcYFIEGJbL3m!$uP8~?)% z2;~3T3;d66{Qprm-lQ8Pb>ba*(@`*J?4me47d2+k+~l4A5K3# zQWq{_<^^MPX+$rZu%bw?A}AxvcOj3N?%dyeM8mzmyOlnPWtOyKmy=vsDB>iHeOn`J z2x&>7*FqtF{>wpIHgWEbs*k{`A}m(#t#Eb}?a8w4*RetkJnv6TIIWZRJbfj$bgR5U%nOY&9qHuACq@r zeV8Z{U+DhOJfh6n*!vP$`$K`n-@&5PDm7r*L{peXOyLC>7S?uZL4&p=(F2dQZf7-! zr!c3ecD-p#NpZpUzX+IQ z4M$vNEzlM1ud<>d9AEgPpT59IT9Kc`BC%=qlfJVQgHL3Ix(}=j%!FLv1UuMJ>X0RIS{+|%1j}-CSrd~kNhj?sO^wEv0{b1c%7wgbGSUfgO*Ypk)paCTd9aWldSm+7G>-jt1amqgZgAy+}ChHgK@gu zSL4I~i^-QJ1nj}ALd-}jDk9el%T)m{z}iBgBP4Hr%b z9dGT54}UvhPP4B22gbam6K2`&qLkngYh7obH1hGX^~JC=$o{d@q?pS^b!seyyc|;h zXXce}&Pf`$*Adn)s4^O9q?(ox&=Q{#^X#@M6^lhhguQ#*->GKem3x`o_8x{$qrEsWNw|h_#Q3(|to5Pnaci>0Y%`?4>HKGbW;j>l zbR{YtJF&ptpo54aDb3}5^DA}ivIZ(?*-y0Hy*)f_FSY2e%O zj0s8&t46f@F0RM6)~(=nTvIxD(#zr3lg$y=S(ei!twb8p7G}r1$!3-92j2yPt2%g~ z&1Kb}boe5(mj$ewUpX{88qsfM*&EPPsZ)zwQa$OX5;sFOG)g}2&~|RW^Vwnc4L+i1 zs_EqSm`O~`DTCN7qCphtaugmlX4tGkikbA3cB3n)`QoNT-4fHrjPmAO(#d+bA%kW! zs%Kr}Z8_yxQ*oWPF=bK=f`y_Y+oNlgnYQ>o6!kb(jf9xMrl4fqY1p^AnD=4JRz0ra zY1HD0IZ>+PopvD9A1;}!(J{6wYM*2O#-2WlRaM7!dZd!JxlF)9$3{o+C+spS>N0B* z8w3=_pl+D{dzWY2e=#Lb#R9Owxj7-x1bidRkY?a3csVJQ{W$QZj%OqYtG?9NwVzfp zH)r65^iyvQ5|Fy3-mq8kjpG^huaaMBM3nBoNW_^$#Df%x;S@}Ds23AOxgn9Tq~v3p zDPq^wJv;V+J}Y`1mKcYRkVt95%_H1~a=cZ26EBl{-Z!6Yh)%oQtT9Qri1SvQBvij= zSxh#)Qg_7J)i*1W2@!m>f2MD9)sEBinz;o;g2(I5g4Aqt8 zx#hhLS{8WgkUhzoxjAKV4$0Q8wz=UUW7f|WbznA1drwX-Ig{;hcK6;f&wUWd0@~P# z>WWO#1N@p-r75@jDu0Ps<#8ymbowpop4bnpZIiW`|2Ig4eqc#VSX10_@}i8m94E8S@g9ykoGM1pp*e?ESqSZA*Zv2Ffv_CzD0SaxU1Vq=nq zFVMXEAbR@f#x6E$w*ihu)X54{^sG@ZoVPI?lh$>$NX@r&!mYDRDM~a6-&_YRDr@-Q z>pg@INu%h`#t-2G5&A>w^fQyJ@$Oa8uLe69V{8a05IbU~S%b^XV9Vmj|C)!bU2Nx4 zH~fG8*>ot&U`(&*`q}bg`~ES$78@;u=JQq!ds!tM9`qtHiqYmtmzGrdp@8WTCOimrJXNbCG87j^`Chwp*BbFz{{QUduIXsza@$M^ zjPIwT%-*M1hDhMKFoU0JXVsGxY`Dwy-ffwynbJV2ZesoH+_!?o%9U`WzxNt>d>@YX z9Dd4TE?xa8fMF=CB;+A>jE2~u-3Z(k8^Mm9Cs8vsB<~Jcg%uKZ`Pz~!3(1)*D+BFU z7R0#FuQuljROb0EMD2q?mDK%ST>J>BxuilqFJDikvmEx7b7bK&>lhV9JeG5?#D8)r z;j;w|g03 z2jtJubm8z4?aYfp0Qzd z+FT&n#I9_^qTh?n6t44fmK?e;o?iN#SJRg51gaMsC@;RM%yF#^Zc*##B_j-!H8-;# z7=EM{rL=a_kCfGkteYwg3Q3b$ec(X5+W^PZ$@ENqgj>Rr0-Wp8KERQ;TZpNG3THWa zi2T4i!}U@N?D}!Tp0$}ZPiLmd#G)aGw+(U!Un{u6u7~~pBx_{44O~wV+6wdA0ZKUM z>$$zxah8i1R(q9=kM37bzDViXy)wA&KjT25Nn`Uo*WuG5kS*J|y(bPGFWV~~N9P=x z%str&9}6>AaO1l4a5UrI3FBvz%{>(r6?66!_kLy+BZ-r(uFbI1)aa_IiraFfx$Atc ze-?VAo=o1zl`Q8xc=rVd#VxY7w`K9W2e?<&B0V;nlfOP4%jC>C3$an|aTP%HCW*Ui z1VjG4h$c@|^Ssn#^C#$^wL6*`rULvg*~)SW|6JB^RDXQ!9t0%~<58cmhWN0x@UX!2 zn|zK1=j(1eDNnx3;5A;y=Fx<fi+{tlhy}N_Mb+q_Q^pYnF7Y2Um;kCWg*&AOu2D_RRETV7mCgKf? zxaME6?Lqa6%_mM72-w~;Iw!>57RNA1shll$wD3S%%Lb|MJrHBR`RLs%lbkB!``)g6 z!;;?)AcdW8ZQtEp_ZLyX1aZrb1{SNDSd$^);QT4=)rqwQG&R~cEN1iYTyi=E*?GXQ zY4+6D>yeIAaDFm+K6ULeNI7q09+^qU(kO0wy7juA@Xm7iNP-pjZzXqZpoy`Xe(LtM zMt`}+u_2G`vkN%+>gd1)et!M{Dw{^Wb(&nWF!N`bFet~|$r`Q%?n?*3Z0TD_Z@zG< zf)Q%BxwX(QnYIR*is1!EIYc@xV~2Uv%Jvr*CGBbU*iw#3Tc9bO2;L>GRr`!G99Yj; z!CkJ2b^32cyI$z^F_KhYU5ldM%Q4mRei$o9P<&M#H#u8%?(95i%Sna!R;b^V_^lBC zXO?A?S8yEsPu9rh=DGA9jD5Pq%iD37z`r)FCJ?c=IqtMTI}zNe zg1{9ByqWZuIaw*gw{Brp419QrbCwHS|prQd|wf?liLgrjP8fLb8x4I`&cpz@6 zOl1`ApOBD{p`K)$nwM^5HCBtF9u81PCuih_n`x{uNjB>XH%um%S7?%Z6&VOh+O9*S zOS|LEP&58cQ=NT+JWCHL#dNB8z^leQ4KycW@{2xKu+jXcfE3$1i3%3A?Lw1t%$X0n zaX4Pd{fRb=-mZYL4Qp#zICKVbA65pz%GeE`l{=(O92nTwk}Lcr1V^oX6Boxj*7V)K z#3uS4HPG8yWwBE=Df@@7GDrkMJOFR4&78I zFH&;vA+2;Vfp=PtQ|$l=D)Fqpx)6=YoQT z2cbH3MIR|)`Ad_7_0~e8V@ZT?^5q2fj=jU{ejYN1qW9dmVRC^acrL;uuhjVtee(0r zjT^ENs;4l4Ya49&cVf?s!WYT%-{GwsKY^NS!qq}xe4#lo?C#Fa+)&lJl}f}-XtC%G z6Ip@gUFRMZG`o#i+{AoANIg*2oyk8UEX<9l+bUC-Dl;5o=cP8si|sGza=BtftzDRM z6zjnh7Yn0qTO!}OBY1f^V2xkTI9L|+jQldH`MnIdFv1qW2~(qE>AdnedDzp_-9qTT zns65#F|51V12v|_z{m-{r@1g*=&mOppkiwcT;FSIYTmGX;$ISp#QX(DUBX^#gg_C2aQXx$H8s@|UK!OI-Mj{3JZQrSG{`FG z)Y{XX<-kixDDC#oYg%5glia;zYKGoJDs_=PhVfF*&Vz|;q{l${?10kFJ|}u>EPo!0 zsGAw?>>N55QD^+`ifC;Dn$&I17YK!-PdM*8>3-WBSjE9UM;!A8@id9xE8Lj})%bGU z{aL@)O4(Qhq0B|;fg|~;L{?U`B$L`dpIEDDhr{-n6*-7+`EWu0{qjHsx~dP3FX`)^ zm{lGE1sYxTyrb7ds-9fN(c{9`q>#LIosZ^bAOgrl^)Y;|49!{;#<#n}?PtjTjD-cI zD&3%jS~0a{lAQ3Na`>534yB5GDPg>BBUU!GKxvH-5BQpMVDw(On4+R0Aw#{DB9ST^ z%88@pLN&>~{xG^05QDq&ac2a@r@wn?v>pPx7ACuqnWjh_+eHnc*HZ2Ki`Ufp?(;mJ zygjI!y!@-EHZdF_dC;Al*><7D0r~R|YFXZbFAG=VBC9XG@rxE2}c& zB1+bk`rkv~XfbZNI82WM?Cctm}_%qZ(aOmw>8c?iq2@q#Hl**k;oXYFBEQn z$H%sysj05ofD_hm6w1kDrN{}`r6m%{q!oIcbbwi4gn@qw%rXgg$D@5&mfapxwHZ%` zJM&_u&={d){}cuCtU27 zOWz6OL%6=CS(E!ql=wCm;*M|sqa6E!?aEV%nJW0EI5cGb&BYVOFYmv&VbFg-urTT) z8tr|jufL>i!tet6hKWh#6CSc*`*mj@#`mAh4B?F-xPev(OeRq!pnbBGiobi$jBiWr z$1eX3);uU-%}~p#Gj2aIQSNkNTmctGD%^4I2VCUb=eW29GK`Pa;iefnXd~Qwdh%q% zkmvk^B!B1cOrzU7p>lbvkL$`P0m0x-Xb<1my*w%?6iqkXg# z@zCk==-_4-Qh$k4?~NNWMqgl}gj7A9Dpxd`7{F)psVTs~N)m%=@>Ee#DWE&r+jlrb zylu3tZuTOEQTAW_vzDK(Vh%a{tr~z0+H6cS8*~fFB028mUx3mJgZ64SHa1%Bj#>|) zQmIuWl0!Ehzu72Jb#uBsWo59Mo(NMFH|!hS*<7+TNeeo|O(K#y0tBp0DrwfPE#|;a zI^Ym0f7XJ;KSlUZyRdVv-Svmbl_qZEjd7)r`7jXqpMLQ@~fr(u+ z07DD0fkyG)x#KznF;0crv+a|4O9;P>JlfwM#{an+M2NUJtjH-s-#q#4Id{14HMGx9 zVd&D)kTL%?*zV_kZK|TJ+``T5H%i|=YlxWeVQc0W*Pj>tB;76g`ueuc&G9~p*5f1N zP_)I*Q+1D+F)HxogpF&ys5XO_tcc?oPdN*b(iR)ea+qp`pJ+;gW#f9+8&yL;(^2k;pAb2=Kr7EKfD;YW81HFr4>dU`h36?`t+ z%Q1O!pO`Uf)r|>Z?#T&1ee&csYcGendJNCIFCwC1bgO z4QIy#%5Y@>p{$hVT9oaO@p104>Exo!eQ;sn0xhkFF2vl(p2OKBtwBajk-=Ltz~TuK z{X8$V0B3K?mjiVex>IcoO$ZZl^k*;_PCGzwR0h^O#fx3^&qoryy88shCLdEzDcKd( zLB>W?$N>dPx#3%9h$Pa+UK4)@myd?zQWqutDN=&>8t0RbyBR=_c0S==P0B`0gbXeC zHgs%wVilO1WrE-IIXD6DKtTE2Gut)Vr~-Moe2UFup?=9B$I0k1Whd|j-cl~6um3Wa8DHL2YLEm>LCuITPAkPTq2iN#Ql zZ+-&9^YQ{4l^v5wbmW_@$@Od$!YW^n&XaP#XSZeMyh&HCq3F&bSE9z7V_*K!u*UTp zngCru((e+jKXzJ0%5HR=;}4P_R?2R$iEw;?IJq|cm*ZqqG`{P(fK){f1akG38F~k> zh-~8UuBx*_NFpy=iEetjo8G#0U)6|jOAYBkN+lGa;=ty*5G6S2bF=zDRbOEdJmBs3 z%ljLbPQ%J(1-R24Z)!N2Zw45}gzmwIuhqNehJLW6ZH;MP4!h$4I$YnqZ>v0m9zUb|!=`^J#Xc8#bSlVr`qP?J!6U9g~dHQ~Z^rydVLS|yX4aNd@4W#4-p9+U_Rqg*GfZkj^3 z@16`=`bSn4#wP+yIftwvR^rI_Bqt9x*W<3gqT1Sg$B{gAD{MhZa}9ErN;xy&z18PI zEo!GtV(=ku|BJqE`^)y8L$FrFOCR1N*b$H=%z1lzd;eFBF!fJr?nC!animwIg$m*B zl_UXp6*G`tMRyr%@-HcA4xrKe^NT%@9uq06`4>!pr#m2Fy?CjJg+zxpYLsL|Jftl`HdU#V=#-s_j65NIH9h`pr)J3uG|Vr z-uLt%S4KajO-_?Y0U#c+H*|T(8fY=s@-cVaegFrHUB>Wo^+ow|;-|!Y;F&T(MGj?9 zw2!Urku@Uq?OrV4*8M8(AGr3m%ma!ul(g4_ZE3%WT`cG&3W#AuUcZ)EuT{J_+I3$g z@~rrSeh)igydH3wTZ@3#b9v{|kMdOM$Fc_M7lk*@!`BE_4!=Bwp5sTHndLexyxwo! zwWv>E`X)4mtMmatdR-1!010n`TqeJupu=MtPr}QM_qF?zdza(>s1an~y~2it^`Hnh z4p8Vp!=_9_L z@1TltZmtE=-E{NXrX^D|h7v)7$S%Dfcn_t{ZjKE{ls7&Afu)| zKM=rVgVnX6_Vu8MIPWNUAa=3@6!2xbaZr`xEaRPS@>3L^-86dpNtnfW_|?N zRtIBSee2m+Y*JFT#M(UJ=bQwjzQ0L-A)~oJQ^RIr0Tjb$mL=&LLJ$jgkkj&kGJ%*o zKCZJ@#H4C@IZg}nXKZ;8eZK$iS`xeL+p6&|#oZ0|Okf}%#L^i?PTfxu4h@`K!gG3U zIUeK-miEM8lJ}o^H0`XOD(_`E$B$b1io3>=@mm;D!k zNUyBcRoxpn?#!Pn5G&n6teLvxC6b)y3JALWSHa&1WF-~1oj%YLFUV>hVc(B5Y;ql0 zD`G*$qfv?jdrBdzW)f-C)Jk9H!?6HPBA@4_J}B>NiM0#%bB>dJ`@3j4+vH6_mR>7b z+a_eZ5CQeT{Oaf)c%MMcoJn-xEO#@EbGQJol6nq+X#D?H>Mp7ou)y^?g8A8S+<8_v zR##+=Bu#h?B*~4S+0eAnZ{08!6$u|g)6D`UT}^z=0y#T6Li=F^Mg;5oo3XPPg%(LR z?qToajBXYwJ*t=?&cS=gnet_afqzeKg80{?DT?+S`%D(k{#CqPqX1ON1)q4?G{v9( zq~?&-$KRp^$h@2hGo7*H(0mAE+Y>LqeuXbK(Ep^vdB>ItI7(}N%na1*W?&uQNO16T z!`9Zmzj)68P0N=T0nLegm);qX2h1<1br8FJLUz4ib5cax7^~!C)m9UwA*;3TquJ2`v_axuZ40HEg#l+A+$}+4V+$Ox>Eqy??@za`4F-Llfy9k6ni;?z z-o1aT23<=8z`=8)x!uXGVAS(W83YQPlK%5?HM=T_DgM(mke`)7bQJuhCDf5H^TAtA z2GCzEkq^96e|MksP^ZkVf|dhF!5(E6Hgw3!e)Hkb9!LvUTQ<3~dxKQPoAd(>t4`;- zM;=O>MGgAK*;o_Fl^apOH~l}qV8Y?s8hHbo26cJ6iQ!KB~aGT;f8CJQ+T zYq59RO9M`6wJVi8VIq`S=ipoO*YqJ6Wv;RcNYiN*1Gvk9eohh}Z zfIxZZEDVv#%x zlK{W0k=&wY%Y&Ij-4e|G-Ra%rc~dr_cr)iSt|oih*_}^zIC!t@JlFV_8*SOiZU?UR z&l>M=iWjEf%ysDyyV%C+*znhZ2&UlVuK}RZe_c>Pkn^ks1jY86m!KX-CY)O!?I{@j zPaExW=H^>xF1P(9Ky<4jLCgD>i4C6x)tj{~`XN6&_U?Sdow@_K@s@czUZg(<1AvE{ zu`p-lNQrYvZWugwO2V;>F=PVwPzGY|-r-9wk$N~+OJ+P9YJKs4d0 zc9^*Zf^C$LABMgD4+ZDH+m?X{MiKi|M3d?(2d*uV%p)O1S?tOrXw8lJVjtdPH8cJ? z*s)=0PJo}l`X;;L>Y*l#=+DaBLb^`GNMUdD@D?ueqC-&XT> z3u#2Zo8$KKLHqG5MHfVqBiL@QozbgxHJ`)e{2VeWy*nRsA7*nU(y%3SbDRvP$HbZB zFw~WmJ&=T7g#>V#z>xF){5fFJ#L9Hg)n(iCO$mFBwHc90Eg5 zg{Rb2s9W)02Oz2y2PTN~>Y6a%e{1e-nffmeg%VSRa{tH42-9R`v?U&J@BsHZkM6K# ze)UiW$7P7DKFEO&3MFd;n6Z8EJ?K|lFJk-!VT!_Tv5vR@Y7T0t%M0Q;BG`D~6PLb+oA>lg-pJUz49;PaIU;T&Eow_&2WlG zOW6v@J|VNo%|0rN$U(ZR2@XXT#tZ&F&a*!${yNp5pW^}4ap)WD^}KW87^3C{KA7{=Jq^~fX|RrjkMcOgFhIu>PY(EgY0q3o~ms3*{VVXKSWh`%;A zf3COmu;cK%kH3!gD0{{<{}q;jD;>9x^td)NbVa%ku!tz+IpS*)K`xg81`QI z%O|kK*!k_(A(>~&wtA9wwt6El)_o58lexixlK%zTo)Qu8wv2sxDk*Sz{jwiMWWAqO%}ioEhZobNZl2^i%qHo%QEw() z0=x+M=YzYRL(7dVes5Qi^ZSI&K4Nh?T?_zYmJJ!{Cn4R){R4~m7Wzys<;N>wo7!yc zlXBc$#YZ;z=1Oe2G+-fgjJpE&+-#$R)1zQ5XpnKi_>Can$#-c_w;l;CkSn^qKU?_z zc}>?RLT8Ij2VeaYQ$28D*}f#)nJ zsXX0Jp#IST^?fRC$2k*%81oT+#K;9UygF@M&2uN^))$j+n0@#Dvld`ZjQR69sOx|< zGvJ|)x}S{MbI1wfX;bULX19eq>rXNAO5i0kHI8ilBI;KWg7`<_)tW92=9P1(!O}239HS z+3^<&)GK%%0x&QuMjA`G#5F$80Cl}>K5%wMg)Z^r?Q@y?kegJu?D1ls5M&*_PV z5XcuHNz72?upoIIgwLq7Z!eq9pFwc+xl&$;oGOJhdqrk21bhBR;-K0-p{twsai4d( zv2*5)y``tuU_*NR%tnat+3L$GaON@jo@z7%lCqub{=`EFa$8{#=dcaT8hW7=U3$$0 zoAOq#rD#ii4UovA1Ik&u=XMdJSa;^8>sUWtYu`=uT7HACxMhl9f^A>)CYQl)$-8ht z_N9*9HFP5N7N`*MjOpwVBlU%O{pl|!h@T^5I-@QE8e76WisIM{6Q9A+y1y>KX&Iw% z-+|^NUwNv^z43L!7q$aoAGn>>^u^e9m_Csb^zFKlhmisC)gq3XYI_MH0Fi6NU%fh! zN-sTBcBdoLVo+1w&U9&_NwPJ|q|8do{LCv4z-NO1B^M;|P~_JduaT*T6_3oN2Dv{OCs+^Qh zaD+hYbGp43b;)CiafYi5Dj%;#M;WnI=M06ks#KwdK>V+GsRv}u_lrkz+idWg-qiZX zy|U{74$6_d=MkExf=ze-UA;=XPgP@^BHQ5R<18ztIa7zItb3zZhLO_!zfJkTMQLWN zCx`P-HXBBAms|n)|`# zA;QuI)ha~R$Sof}I*T3oJ@5Xk0-Pw|_V?&f(Z%5FO?-lMjFUYX0^E7eo3+8XE=Hn%5ABK7@uVU!~o! zgcs(_h$uX?wsnYvrXtq6D_6~P07c>|iM<5*N3fcM-;n&5>|NbE+9?gH`fKA_j)OQ- z!j24SK7~qHNUr-X!%z3{hPnUCvt#=junxS6kQEbydwkId){y6Jku!w z8}_F*Zsi$2%GRFA-a~c?9AQAN2Pkg`K>e#BOm~1o5G`9r;>m+vB?rYdyN&?GzHCw~ z)VSW^Ka_#*sGL~p-XcX;u7_{gCbT{I`jaoHe>on|e|?`+r0TR!G8lx|d0udwfs@q_ zo>QSsjMqH;!`i^{ggtDz!QcI@oMX6LPp}~(nCd6Gy1aeTF%Qn+h>^Q89jn!iRodU) z%9k|exIZ57jvEG~{~hZ0OHXzZUvVq={}p(RP0D#zt!nprJC1c}Qn9Feq;5r}E)K-2Sya!>^Dq@~Kt7pla zAzu%nK~RQZryCyq^M?!w&^sKJ4FLXk;1DffOae zzbX#@`?qg=t4wy|z@_EG(hn;S@|4Dl|FMGYF|VE!Ou{%Pzk2zDiOeu}eGoguGlCb^ zJ8|5N%IozPSmn{NofWsBEfUW-6e$KV(oY^lA)NN{046@zxtE%m^EVDWPl7+^wZK?d zY;3vCe&kxn;$QlNGnVNE#hv=F0~`(%0(q@p@&f5a2R zMELcw3vq$K{nkGMLEbN(9g=TPoM*zX?oV2$v5yD1*q$Tsk+w5{F*lD9m}F{M%)s_S zdtW)9xh8LbL5DR#6xM8FBO^J_;iudsmsbl>a%{5F#iu>)G|iHISo?#3OLk3MTjIp4 z)t5n5RZ{INhd&?UcvTO4`!OPhu=D~eH0wE(s$hk;1Znn_UNwA`sg(Ftm)0PBnr+ldCf#S=Sx61Ci-9j@9Y}@wp97frV4XLz!hn3hj%?=qVyN{m4YjC-~ z0^~y^LV!Eg0RlSq(&nZVkNrpUZ9Ms^P}hnY+rCboR;}*hiRAf@A+qC#rAvM1Ol|ik zAV+N{iiaXRDT`17N(DNbp*N;VN)g9u&hnP8m)fCx$gA9=5Kq-y9`_e&Hnj)~3+UL1 zO+ZZQo{WYosgu>nz3~?coFy*w72BRWGx(vZ4YiPGFlV{n8s}7fIWTpuNkeiZcfxzI zL{FnfPlo5V3`M32MT(XEZtifAcVbzj=)Fv4hr1iCGJ)&3P$}`Xr^SQZh`~*ej7$(id z)n0pL7iD<6j@T)|Kv4%RE=avGaO$TA0Yv!~NphWlQ!k(0t^3_=^MSb|FLv!-Ry^>w z^g-P@jclLqF+cYBHSGJrT`N@M?j~=fcxum$t1G+D+c81~=Uyd=Y|eSIWn-8_0e1xoKt1A%o5z|m<@>Q(TK^` zH;rS05d9q#8P}`&JDiu|@ULa|eEcj9$)BG@kzRqO`)Wd@H}bplq<8imPs^cT+Rj<7 z7B?{4F{{33&M8q}^WR01gB$boA#%Fa=oaXYH{`vJYR(d;d=79ya95gWxfam^{KL0_ zJsbacWK<5QWf1XY=;#MCQNBo3HO72%TU)bgn;GVQ$JhC}yfO@(fayZ0HMsMio4&BagEA}p} zEZ%5XkIHuQj?t1h{?k%i$DSssEx4k_u#N_-rEa79WI}-*+-vwpomaFYYr&`NpFjd) zd}D*y!N_qgPqsm{dX6QvQRVwc-uvwPmNo5PJbZR&Odyd_cdqV(IDKiu1cmGK#k#4^ z!w0n2EvQ6jq>){^bmcaj*oTLz4nmht)7s(9)dqnA2y=Z0;|>YD)x&-_U?s25o0}$l z87vkIpYV}XR}kchKF@=-%cEV9wSaD!d)}V|arO)$=Q~Utlc10Wx05Sd@41MbY{KGa zJNjA^6hGd3Hu8hwnPH_b5mmhP_NQI`Ugs$m1>lXbDRyih?{YR8*r>Tbm=vzMoUZ_T zCzbzwITt;uSI{`-Vg?DIz7mGH&HniJ(gLr;8H*NSx~LzW*+8Oe7I~R&@EX+f-K*;*?FZshG2R zi~S(324sJ_JDtPMoYjo;$ZDJ%jC`Wx=AxCSLM}$1+I(uu(BzgrLr=2q1T#lqen)2G zyKPPxJF;u)HKU)wg_1mkcEjT4Vw_q3h%4s^h16E>bCb^qZ@#uQcAtP>gRu-g>Ng-Q zLvBC<$KLPor$9| zK?Q;?i&Er*lz*B96_Lc~^$}W!yhN=a{+QC+~#e$P4_ z2r0xe(=WUtN_!PPI|>+ON$&fn6y=lEm7Umw*~$)8%~Oj+#7Yp}j@V79I!(7S9uuj5 z?g4-J4F2XCrH|HfP5`@vE^%P=c4DI$NO)IMwsqS#NAjU({0FJc=I$@g3?0AlU>n7- zBUu=(V?f2A7Ua~OhaRZhW$t}eO`L^0#mA^E5_M!(s^5urDcaLfZg~kYL;H7!udZSF zFkuBB)r!T=44TRt70;Os?Xs_NbfOihBjJG`+v^0~b1>bD`)qB{E8BukZAapr5+)ej`)QQbu=E z{Mn&h*3$Zl_JOLOj$VE7BT49K!HMCk1l-+SVN=D-+T&$bwV6}7v@6R4rs8i3j)@6H z2&{&~;j7V2x|K&%M#>~pT{IAeS;NmS#(OrS@k%x{H3IDd(Z;zUU7g^~FwKKc>P~!) z_L;4r=M}GHdiG(EeT+9ouv>Jsqai-j9}%z_s~s)L3|^=tJ^|EYc3-{vjgpeIVo!E# za~uQCOU>nUIvLf+BrT$l-?H=U<98xQqLDsLpVDq-9dY<)UaT?mV+A5J#D35vh?o&V z*McfdN@;(s@x~XFEY5W^HJD!GPjP8(I4^oM9hs=)J5o6BuYrCzKvn8=UVcLsnWU?P z3MX}FPn;Ucdk{mpBKe{aP^z&twCh}7Oc}|i<8<~YA>BBXa5mT;JCc@1-O)=AqPWj~ zlmDva^ZM9v3h55;Z^O4V=0q?Hm>w+E0LyZ7zOmg}p+;c)!G1Ltrc%;VR4+g>a7?GtBaWE3rd~FSILM*lbMnVqPvdyyX>Z6)hp(r9Zdvl{*FWsXckPJ+8!WD9eQRA2 z*vc~f!a~y11S41p^P}gKN&zyppamwG1!daYE(9=+O=|c6R4FUTLW$}84u0_nAq4ZxI z5P!KOI4J)SDg^a#e%v1fmyGg=*4-w5wvW*C$y3 z&!WOrGcm^GlS#S0Io$zMcL(2PIt%(rHmHbzi5K`&qyn;ob0rNnASio7$z-}xQpY4+ zBtjO0J?*|ZqCt~?YVSe5@UC;t`tqBdT@QU0r+50|`W8>I3aiKVf`4Um(i&PvEq%r& zMM$Cz;i?=$z*!CkW)3Q;S}hdT2~uutuljCMieJ3>VFGc#Z>)|%W)<0liK^6;5ZIxy ztP3@0l=q+jJJf$)Recbr8{}#A;>Dm!+ugWe>df0WR2c{En4Ix)6P+rd)wFF|J5mBv z2mD|jOFFD>4+5o5G-1*1;@~D{nCjh*%vS|e@NU#?D_48KVs94V@ILY9FUy_qSQ=q# zf`@p+j*#p1LJf*lho*F{TsRcdemGYtXhG)yDWASydsgqzv<3;SQwwT(dZmq5GN<14 z)hc7r=bc$n8|W#|-VaC_@3}A?b#Z5c5paFE#9|?O{+>AnADJ@JL9*JBkppK=)gd!b zw6wf*%XYV8d^RgTZ3`Mx=(AG(S^F_q3Jx}{)$TuE@tIlNac7@fu(DT1_|M>K+T|Ch zqB7(9H*Gd0Y^g?j2+e-Bt`qRaR-{x5yp6eI{gN>AE{43vy!>NuE1TgwG}%k?^@%&oje&p1=66jAkwG=UAiDY`AeU8yP zbC8e(m0ZwOm(hOLuw7@bMVzrZ4CB?-rDHfwqY7EXhsX33x^b)1)e^2dQMnXXQJ;o%jU@F&3U8~`o`Ov!*-O;v zWs$u5)4dl#2^wus${UiN5kir3rU8iFQD4U`|ERo}q9b)iS|?QXt9(ttL5FkI;sWF) zOR|UMPD<&s>wVRRbIwgCtLF}qc;`G3ZJknrS=V+Y4CpXQxn2wU`}{6S1}_FaJ|{x(7|e`lWq7z{zX1kzFl^(n`)Bf3Dm!j~3ZrN>*jYe4#`>@v5Ov z%*3J1lP_HRR+^fN?#J(xKZ_0bd!Fq5HG`a3q?OZMbou5DpYPp5A7LrdPHh-Qca6}U zH`#j#(Ws|*GY3>ohkSVgydNr3#UN1}ljcflhJuGjGT$Z7JWJvCrqJm#--c4g^GI}& zDb(OB>m|sK;@hvwpNe~t;SCkH!O+R^JrQ?DL6GNfEe1j+7f7^t%%(U}p#h`Z)%6q4I! zXR&lVXR1(iqrpDou&yDG15Ng)+?tLz@!~;PYRmshw(K5yidu*9V0{!gnD~qsOmsNY zCH%2%!q?*#LHNV>;UgnmqOOUG)y#;xu9}{qYI_D=N58aXr^}6@LJt9osnXta;d7qc z!ylwNvv#xJ1YRu-yn4qA+x}sqo=4L(J~T`)W9Vj)gZ&v=ndD(VnVvFwPwz+`q9(Te z#v#xAFUXmeh(n&5fxA9WbOy1$+2mg&<;EGDUbMez68+BU;alFr2mcv$iAKIFJa~!H z({;G8x^STnlcqhm4X2>KOyP}9qG#yUsWTttZX=JLEp62LXFksK*`2Q_0`6k*%G~kj z?~`{5WQe-Mp>7x6Ud1{3tkmjQwph8EhYKGmOv%lc9(^J z^Xx4X8Ls{`mO1qE6^)ANP@}U`miW`Y??rOg

K)#Ix1p_ zF4B0=U5&J4?UisaOZVH4hS|`QtfHpfNX1DJ`d&%xiknxtEIs=)yeA~>B2t$Vyh9yY zJ7Ti33ByMW`ph4|Mf()W9H#3~4ZN9ArHVE7Rr(vdb`~l6I9kO87w1d}pHPoj5;Lfz zY%jkeJ?0l3vbYoUwpJ_n6=D$k>Y#x%L8|@2U42k1h2$R^)aaaIy7;u7=Lr)?U_7rK z9y#bbC^8Pj^+}#T;?9eU$zj!jAQ31`FsyZEgqeK5G@a)_0G{dLnZ+E*P9%@j?g#(@V`A_u?>51((H8(*T&+e4+p|DWd9o7-blQR}4{w;~gdvV{Sqxd}29gr<*`~Gz=c4{OhiY#WvOidUs@Cu-7tEAa{9z9ZUM3|7 z**MYFPMTS+VWHB%|%TP!n+KfP(ZAYC;G}?wkOM z?|s+0>)!iYzd!E%p5+oa=ggd$J$v@-*?WKXj0~zJVbfe<(mn9!?e^swA0N0!x@llm zl$Y{r*f7`dRf3TO(W1rOR(%ufANe43<-C<1fndqeA1f-L=UQVDH+G;}uH|v&^ChYG z*)w+WvKB;{`u5|~HSjbmez@ieUk`Gs0e_5tx$)+X3dloB?Kjg4!ulmPJrl$i#9AgGqqTEa!gl%MOzCYxPo_*TZDgwh; zrG5VZ!~OKw>Sk&yN({i%T^Y8f7%Lk#i&=But9Srsp*_zKN;58jVqJkS*XSd7}-_ss?w`b{u=netI#j0zKamE zJbf=n=wV0p?Zdh^kj-Z~3Qv_FYXX-aBt>E87Q2VO9G~2e;;utG;^9ZR2JqASzyR<+ zwR>B)*Z*_z67|T*+5Bm^M|@1*SvJ2d`0soj&PNB51g#^3w&0uzRu6m@%?*Nm|1g04PP)$Pr~}lso`)hq5g+@@gOV0yywJ{ERc`d314PQHLfcd2*noJ! z;Q)7hXSd^RiMm@3Oqp#&=d?WpY5BHs+h&n~=0k72fpc7bR&*C(7}eWZ`tsPx3kM75 z?NGM+Pe0cug71kh%-+~Ph;uXx*dhJm-IikmO|~bG?YLeb%%3rFk%u?^)IdN) z@$^Uk`S35kBm4K|ycMbbwN;~7cck|pS*B@eGek#p6GiM{C(4&yr*YX4-xj_`>guzaf4KW2Jj*JPpOLopNabCU$_d}9RaiPsz7kIQw~zo<^V_n# z=zo6D$<+Jz@ zwJ*AdIP72Q?dJ;7TLCKCeS((-sytp>(9!V98;i=zTn0~Oqldn5-c$AWtx8vNi$7Iz zfoNjzd09O7M^1l#jgO$+M1my6q^6<#yHh{W0awIHa!yd!F4k4Pdp2`05J`!CP9#s4 z)eY<=r~++MImP$<^>>O^npb-DQ%583eY73cNwT_Ln71@#6X|x$c)HN`(RC{Y+t*00kO0Iu0aVtVwsiA=b(2@1&1m9C0Az!E0XRM zu&l;b*nKjzHQ?Z5(z4yV09SKH9IokhwAiPJTUka?AB)8&KE`F^$ApUT!KxRAN2{*} z-I(#MIY;gQme7~h>d$Jp4YBgn?uM%iX$Uz%2{!6wu(aZU6ab zztVt8AYSVz*jt4=hbNy?e%f<%$T-!ir;y~KStO$I)3ED_ZR>c!9%H@*p&tP|*vDmb zV8E!4KxTNm<;FrwlXOs!&hWxkwrvor!*iaYnM^94h0v8CxON5P^a0gnXi4=kAu!BGJa(8tMw^ zqn6rIhj4{iQvpkzFI@s1BJ5))I*Xd((gNWAmkTv2ojQ4Jgcp8oRmtsitFmhRx%r&t znx^D!H-fQyodR5crnY&@`QaXCHwjA#X=mR+d31<%Gu65qOyJC_aBN~Q8qmvXmE&X&~62dna)u{~dj^x&hV z6eTC`soQ=g!0tFND&$-plZxNr!Q;Kvmd$$=YeSvkXZ_$~=+SNPdTng%G7O5p`_*8jjbmoV5pzOziXay!+}0ZHKl@+?J=3yBA|fKkYfs1saad zSDQtX%eh#&SX(}x5ba1B64UqSh#q0@3C>!gUDFO4;$^U!Mk_*((_iZS(Oh&h+QJA-fW@jjSRXeH_{P3jS3+ za^TF}N>A2Vx$pVYC9@AId*=_q#oCF?4ws}yN^5rqVb|kUQjY;Yc9O$)Z1-Dha&^oj z(zKZcf!;^r8#m~7txFFb`{ol!;@bX7;XqsT)?>_DfF3>cs%!KQWcK(82LcgoX!ISrN)jIzcg#srAA$B&IH@B}_zR`~ALdBfmb_=NLA5P2lv12!-($G|OPKMMO-4Inff21_%BW%tKyXPNSn-0IdruyKPYem}eSIXkkgC-1br zg=Ph5V97vg9^Q5d!!{0_crh-0REa^d`=i{B1MJn3yj=;N`B@*|y-XfbW;9l@>#}lX z$NrF^=6IB7Ul-c3Y+xb99ivXk_%JFyFz%Z_a-|V}aMKGd>*=%S&xbNGA{#CP-NRZ< zO%3M+vaNXzS+>1^-Ojj|t*pnlM1dj|ugH;3WZWY@G!Of6<}gGo$6p$YerUM6MZss! zl;a4-t_Ia|Mn~hG{T)xy8OY=!h27cRzU2aHJR9Boz zKk9pvhO+K{G1WIWwxc|x5{hT?#T;^B&Q4K2Gyd$93nzVyS0dDnC*1P~ zV{DsrJ7dINMIhphF>(`z2K$Pv_7rVuNKBYF{DrJpd~DCwh^8Q9l35p^t3F{xT>`?~|LR&uc-Mae;wog?_uP3hJRC zS<;IA9qYym_SQZ{sJ?|oLzdtr-g_U%Dv93ps7wQoxeo+YSSeMk4LE762FkJQVm~I% zA2{bRou`%pOgnRz&zRcZAvMM%!W$3OaGc-g(o4=7YQu;8T=npz<$s8u?=~=5c>yEE z*`yiB3pP$aLp!IAD$Du%{yX8%EA>tDdf7?G39h0|!et2{;R?g4^4=w6vKohH^(6#jjd+sFDwl$!iMkkS4TmP%CFrS^>F zb%+(o%7*&g2R^zY8F=h4dKLrR=vPkYf;F~j;&V;jafa88^`&hR^!KpPrnTB#M2k8Xi?l zjfGqBMlA6rcs7kx*nt>HodE+o4V;#!hglD2u=~%ZY~^~D<9l&_&+;U9ROi|RL00st zxK2VTHo{L&$e$VMPXy{s4JOe4 z%S&(2(mdRz(FFy!{sz+2%++QqWqOq^)W*Vt8 z(z>d~o13p`dcY0-;ZtM-@l;TFXuK<>NUEtEY(It1?ci0!bvQl~LO#cRsD zIZe6e+CXy9wNA04@whA>lHUNn81Cxr0sIKw$NRkH2H+(h_K27PzuMi>m9Q{_27bfI zqJI8Uqy$!lSUcd3=!6eu_$Op*>~rCZ^V`wmn{BUB*EFB};fq6Mo>v2o_z}-%n<+7F z+kjrQun6y8%yNp^a*U=h*g*QG%iAN-pyNB}b>lRaEL8OE$i6ZZTt8<&p7S3T;PihL z(vP@f)xe4UIV;;MSt#g%orp%zq)&0FjK~gBi(XL%bbhi+#%Up7`@iq@kB>Teyo4Aj zmMl3yb~K1hJed1oPky7L1d40K#^iF)fIiB8yu$A8H^MZ|8{irlrh4IsM<6aT;y4gE zN6tA?eIHERxR!QX(06m*jbinSk_wGW6^}4%=T-PW9ouV<1fC{L_$WvCZu*>58(3X$ zOzrx?G_O>4+j{)Xg=3wE77&N)5`!a3k0_-Up{bk{;hhVSyY`7)%WO+&&am)jJML95 z_8Iq}rvYxSzWK?1C-<4mpd@e3D&IvEQq7eIhF$!>4Vi~V$=sya|6yl z(ehRQa)eZw48Y48T}DPa;zb(a)-p*RmtV6tR??_DyZB@MOkVj~jq~oKd{Fr`7`!_~ z`D!m`oG$llNqe=2#7HBhsyP0_$@FS8z$z5))|uw#D?25ipMBxA7l;~SeyGj#{?!GI z!*yAM{>K9#3;4;p3w4)6mOPly%0M&)`0hJ48!!6v=o=1x4#&ZNSdw2wNiKxRl@0og zWx&X;R$Ls{NFWR+xKCpK(5_oY54~;XzIj5P+eqFQUZ>%(GeaoPZ%b>Cf!n8O{-hh} zWxzCBkSQQ4h5TyN{JAKeO$fwIR5Z>W%gI zgJp@AY$*2#{jyFnq%ZdS=1h8y(U3?Lvv*6q{2yeAS}RdnJbJTe6^f;r?GM@xfD&Nc zCFC(ONGcR9_wOGO?DRia;wF0DI5xsL9=2<5mGwvr(&_8BIGYk-9!lpviy&5hTF~vu zHoNO1R(eGwq88S~FUWo6*}bhn9QGYmskze;7vc{WxlayqHToqFI3{Em8H`}~tL1=8 zcCqiuk)}yq`o6JjS;80wn?lM>y`bK^P1pvcX!f75jk_$rx+M9&&(yEgE*CKZ0g0Re zTiB0E1uv^qzdz9g#|B%{iEJQ^nIhJP{02T4}xK5EHNkhi*hD#s9d>H7Ak+m)zfX zAQJ7hD?fW_SHh@;CMw5@&J`|FMIt3i)Eo%jgHO8Q5X-Jom3)Tdrau2E*{gdOuSigg zwVG-)|mku&a3inL~SXt}$&_UgyisN(F~U=NdXaGEwjIquBVb!zX8>l`=n zRuAP!_V`g4pZcaGKXNqy7(p}Cw^bed6=!9WfND?O3fC<^l!{8tFfybX`wyG zSpmq+vx=Q^JBs*jW0fzEUE4;Rk*Ig8(2A1b$ZU9|J&$-+)Mta?gtY&!#VqFYy~PJP*nF-b6Y~Cu8ZSa#6eJ@^ds*1=W;s|^^WVM z$el5oEb?WZIC&`T^qUaEI~P?0Qhtn_Z#!U4HS7XsS$iyMg1 z{Bru_z~K|}hSu1hnE5)15osU4n6EX)Rzw?xTr5^{pZ&e2WHurB`R}oXRc@>tUYma< z^fo7#?K;YZkR!m(AKh1_V5X#9K4;pIuI1-O9jyuahQ_$k!%r|Vb@z}fn;zn6p)e0E zV`nM%slbKW#C(%B@{Jh)CklB8VilFwL&{L%!aF~8W+R{-+$D6mN)8kzAiaYSd^Ev? zzy7H++`|{HYyI_1Q)5lss7Qcus^df_aqmUPo96fMa9Q~DWK{r%l>Z^F>s;kn$DU&K zngj4YTQ$PRfzpFD;sI3mmlw)bq%(E%U6Tf`ngjqx4EdkFt(>>oY@90}vhd~oJ9)XF z_pV9Nz>0mP8nDPLJ*&>^D7FGwtBzq}n~n^JJHuVNFbFuo=A7LQrAx3yF-2`g#J|V@ zD~SB{UkckKfZ4=UJ(xsVah|J7Z7rfNmoIqv@Qva7Kp^dc9@gc?Vc`At+*8MkGn`L$ z0CABE8j!efhkRZgf7wqt% z8a;O`G`>)E(P$?DgRg6hRlJ7JE3M*!^%je(p zAbtBUsQCMU1id~WEIt&57&`otT~V>Q4aG3>oS84~$RH6!`b*~h9B>WpdIZFKTZ~z> z%Y7R>R1O47-{hyuipEE?h>EQtPn2@*ZLwB*Eh#5Q9dL(UX+BT?ImnenA@ua=yydE= z9GKL~(xZ2SZ6SJQwNcGHK%qSX1;*@u`i@u!75eo6is1x~_Q>!?mzIwN5dYhdX-{9n z+J`8sUU6&Ah}SWPhJ^W_6C z?$T>UkQk~t&zkP4tDz|S7`!3Pr!=^1&|4B z{`h#dFU~TS8llkrhEc`pe>$5%I6nIdxwl4e)safr zAG65wkk`n3Nq0pgTN_07{&`9KUF6Qib8_`zW>Es^}1QsL>sr?(YVST4~)$YADU*vanAdX%ez#25U7%&0hwTP(o$BOW) zY|aL}#l7ob3kk8eUgvl=!bH$u7!FKrl7=8_iQ791G&Vq%Mhk1VLU>kyo?NdH`nfMb zun&Z^`~pcIkFs(^>}cHZ5S?1fBIZvVu!(o$d7lC8(2m~N0R9+F;b(Ds{P^)blH4mp zPkoSovxZ~WotP#dAY5M`D-iJsArJOj{Q&Eky(4g8lq zS8a7uWn4vrcl0+rG|JPO|C~|JguVrVGJ)wk!TCQDzX;k@3_WFL&>{6=+cRAd-T(VK zskeW+W5`2joY^iHaDB?*q^ha^g6RyOba;4*!Xy&&!B-}CDe0RPS zPUvg%LygvIpL03|V;Mbb`c;uL-Re!i#O4hE4*+KYzSX1L9oB9t^}pKzxPF`TDu8*z zy_^RL>NY40F*5&RSceG9=wWC^16*T6;f9x*cLT)EFUYO_%bS@&+r*3Y&$<|sZtXR6 z>gZPpoDE@mxQC8rxfxvAh{ckErbiO6>&7y%{i)c<4kV8{6^%za+N;_YtW_q5nK@TX#?2xYs9r>j^suFOZN8&P)TNPiMQ zmoTt6&HE$XZOo3s}f^VLh`*e&@x}*m%Rgj6Q%il-^3ZP%V z07CAw+-`kOeTcvA0F)8rJ6*qw<{tHJ{slq1k`q|ttT!Uxz@**>`}G@;^Xf5I`2cF; zv^JX4;x4%=TD%6<>Pi#p+_&-Q5{KK;QqUx30npGF{)Kf_4XntIrbH?hBfUYkLO%EF@=Lhg03djWDp$ZRCE%K*-CS=F}fB z(P6w1kaaTZ$Qm|A>BqhO#2p*Q_GEap%~3cI+rnCZC6E$-m;O!fD{ctjtf1+iyuG=h za-II*XY+25LEzW34NM#lBy7ynOFuRaVoAC;VkeogsOL4K4DyI|@K zkOAsSw@5qtyX+ZV2&t5~kDv-X$4r00i03q|`|eED3jFT9s+(hdUC-B0Av3(kv@WDU zbuW{%0ZN)Ml|-rp+9?wN9m|scv%-39gX{7DakdJYY$#Kg>Us8F1etN~oBSBYf|UR0 z{w4vY3Pk|fNX1L!YIezs>4vVtv;m^J^b!PWK0IqJ1;Mnhfk61!@yH?|H5m|3xl4L9 zB7Wq=qsQZ13Xf*rDJ-_Psk$*UEpP~w#Q|iyZauLNae9i<>b@89=|I5x-+1pSHmJih zm8JE1EsiMjfjd;zW>aW9LYdElYEx2(x5QwW{!CNbJ)~bDYRRc_WEWR!SD8&rIW_rB za^W7ddHk4J@Nsp&5MsYgf8{TT00kh_kQdaehycL>Yc~L}yi#e6v$y%{axH(vA}UAsIWOmO916LIQYuB2zX8z$pJ(altha` z@O>b|W{C4s%PG8A^{KlLp_VKV>vmd>)aw8kqpQ8$aBK)@YTq_1Gbr8-Lfe7>&g;G0 ztThQ50&qiA4dfy|=dx)P=U(E8+TNM2<&2`cpykZlyDP zimD6&!a3dnl9l~=2JrH-GY60X(NM;{)!8G`Da2y^H^^1HL>-)cv*I1Mwx7xW$E71_@f$?*jLfU8sF|2DyLeq!^RIN1)g6Kl=!p4-3L7xH}L{U>=GaRhJz= zb%7CYV~Y(}!PE>QrRNLXwV~GycLiZbJ2xna6?@Z$;jo#|k|4k6nGYYp0>n&@+@vdB zx}iozDKMd**K(lr*qW6fG}xaj;s%i4j4{`Uee@}F;}VYNkp98MNU)GB*cT9Rbb~7L zUhk{R4_gDn3u^=T8FP9Qm_6&4BK$2kx_!u(*&!Grwqw4?xdX%vLm{r~KLGh_80(oJ zi@8!o3hM^_2f{^TQ&Ur6Wg=irVBnr1p#PuC;Yv$;a<4Zy3W@>HxBM$@06MRnTM9D_N+Tl@?)rsP@G-D`Z@#{ z*6lgUFpdCr`}<;O#Q_N*eNH)V`c@Ang96p&AX`}lgEoy?dCtSdxfO-`l0R@tZN+H9GQz2i5s-uieDhX3Isx5D}HQqAzGK~ z@6BZfY!*tZ%*K$Y7E$`b8@>%R^&$7`v$303(-(lfV*Ws3#fH77Q67*7^|+`=43h)E z)*rpC^!}vz28N5UoWnltg;e8&3En?FRxG;8>umd>bWNr9{gD~r7{9%Ji{+nJKxzwc zZXd|*wK}Iof3#iu3tIp6n_fCNo6u=B3xw*&h`*9DiI;o@bDps?i~leYyK%}JS zTPl-JNgZJ%uh9h~Acu#4rY}kv2rk?$a@PY41}2ux9YF)A)VCjSVUJ%Cq#9aq6R84rH#mIC0s9 z=)G#Kg6hzL)_B#PLWKwj9JK&xdIBEAyEf!>0Im~)<(|Y?<1JbzI?GQ&7Cd0fdoKv- z53XmpE<%#vGGhsy+^O_W+@HC}skVR8eLfq3s6sd<6pKv<)H4i>0CJ^fBk$k^?Gbc_ z{kMr~DajWbulZ%t3YA5nz3W=PjA%+D7zDNyqn(E9B$dg*}$~eBk{DK?qGRKj6Sr!fL8b>j9P=(#g8PA7V4|R48R3husLfH_R_}U z_|%}FBXf5Isol($O_?GkpmNbWy2t21&kbWM-H8Z-R0960p_nXDCW1 zq|feGRNEcn z7Sl?1ozBmn26?YeQR^somh0&f$$UtKbuljf9O=9iTulj77#=NvT?sbyr3{<_S-3JW z$O$3~NG0*w2Mo)tBAc6IP4YPcKBbc)1ov!GQ8Y`oGka}|VN=r$rgv$y5YD3ddTVMv zrd1Bci}fHU=(UwCf0u17%X1&h{a^R*P=Pz-6EMV;rAw80xTAUuxgp^2a@4!q9ZkrU z^aVQJ)#0!i<;BoEPtX+ifBo0wX4y|)u}Nvudnh{8i0~(S>H}r?L9(d2P?*x0{#-g` zK{*H7psrp<7wHR4K{(NV=jNv$vY^nR?ja!}0$K;jGLw~ETiq_T@V?Fm3#!CW0;bq- zMWk$~|6-gWZG5Cv3e_hoM>GFj)fO!G+rCNnV&^ zj#Wc>xZgtcoQq*|ubF=o7yeD8RmfVXkJ;`cZr(9e6cTa+LSh-Td~FQ&k(?t*Tb9~^ zc}UJxLFttvn=S70Nq`Wtl+;uU=FZ1bp9fqzrm_%t$sToh>B?cp39*L_f=JFGlDfOc zqAR|z!wT6(v2s=OD#Wft%uS&e7b%t*B<(nDI5;7`15=YS5*pV-Kzz4RNR1@5I=NGc zH}Ui7K@&xMO;Pe~DYHf^-1MQEw>G$}rX>{8#&O~U!f?^H8arzP9(?JOq`ji2 z)EuOLn3Y>FSxN0CX?v7|M^aXavvJ$xQOg$w`R&?rM}tb`XcLC1ZZH>j>NE?K`sS#`5OUz_C!;Mdsl2l0G1Se& zw&7BzSc)B+tYjk0DTl3idk98pQ|N6xSG_N>_wTBc$pe&MsN=5 z9i8({xVXDr7N}unS_;AJNak)=-jrdqZ^t2eFyU)C`&k%B;NRL`4#C6$rC-Ajikqj(@F> zWJ~$V!tWKU+2KIXVY%YX@HKI|}XzYSFS+;6=PtqjM_G z_;Ba81(>Q3QXf&GtN~B0G9pCtmA;n{Ur5n1P-cKPp<53r6JcBv8S;(b?nGfpAmB{) zZdyvupHhHTu~JZ?Y53?GphcgJaabWFQHecLi}TQ8Z-I5Qm~!{QxR}T+ z;b018C4cJbU;QlLN-P<3h+a*13#6mndkMKzYi7v&lp}nOr@;HtC%C~LT~hDAY>MNt zZG?*V-!@$_*kcH1!Vx~EleCIZ)&?qsj7PdLl08~|FVNKHVZ|crr~t2qlquc^&E?sB z+9dfXqAAv~2#=x~mVR4~>g=%5zuT|rsPHml%)LX-x_P1qk34mV?l|$-6$ZKpGX-RD z1Vlj|g#I!}orPh+$vAYJ6istb@-W>Cv=xcB(BjkXm86ZFg=XnNaK|yG8c8iYM5n^~ zm(*VMn6I!ugz7!=KCrC&|_gx~S-o!Gec}&>2dPFhDf>5x<=T{v|BMXzQUD}p=-0ppGlN_R4JA~}0 zDRQA+Vi|xqBxqIvj;}@@F5W#v*}2XZ^hw8dj0b*sd^!W6`fe6RUumB)te*gK3>s-v zHlPA?QM?=M6C~OgvIrB1#-a3Jcz4I9Qxdd^xVXc87B1Aa`Z^rB>5qO$z9k3=kId3L zdZv{XXf-u=LZJ1r8?0GlKzV2@rPd|}CG$1u7g{65L5k4QpVAYpE!ahHyq1CZ#u^Eb z$osENxKtk{BYY1_U~Kw@Z?n#-!J@oAd$7b)J7vET3tjrsCSuhBN_Kk@>Bd&|f+z)8 zPoX-&_McAK(f2lQ4$>3P&{)Ly9QxhLh{v;Wpl@6QA0h53KlH!mQ4@Scft;V(8g8?k zQT?p!U&btRUa?7~8W!3Nnn0h52d$W;C^(bfQ-%jW3fN)*39)-n5F^ijTjKk6p+|kv z-gD^T=}p2{I956gB6)kRZAln1Di9UtIibTN;3-y!8IyaVDtIykdUSf`ft@TtTf#V!mNQNkvYAx`e+}!Gy@<^` zJ#r~@k-Lg6e!h!cC1ot@Lm%O~yZ*8UJu#lvfixJ0QG@KH!T-2q+Vv~c3_7|UyUaIa z2mVLhrUt!O)}}jtKrSkL9ZEwk!orvNb}yo66L;t#dUTG$9=n$6bEl)kR;s2ib|8`C zSl&g2!!+uD7I~%7u1aGyDrxm&_Y58bN3*0EN7z@B3yUxCwkOjBdK2`($;D9P;GkirxKj$eU(LYHT7M2tIz`nmX>5!U zHDPoIq$OW{5m*)Oa12vgU_3XGG@CdfH{`rt45{(r{7Y!dAU>AzCkx$6b%)MUqd{~l zsYt&-!xQ?Jj^y9Rbu+y`@%K^jOwAx4{rZ;}Kh42O-GALE04ECnb>lwBJO9@W=!k)h zH$20@!Dbt8DBk{`X8fVY|4`!(Etr`8hZ_HXqQ>!!oqf;qzd;z?XlDDrBjG%QA?Xdt zR?iJ`Py~|JPnH%I7~wEE(HrEOxSfEZ5Zi3l#oG)qsbjc7)EzA0!#E^8S&}B`Gs4!w z_HAg&wdq=MQUb-WjJR)8`8yasY<*?SiQf3q=&w4dRzIAf8mi3h zBJ~$4D3w`^cbbj+IO~iG+z|S97&!F^{O~NQd?ldZI(CC}?QK_)^)0fPO_|2ni~Wpd z_W_g1y<`|>60@1%{jpgkY1qf@qg5e^$I9TK#SpETJcmWU7mGqfk(qChV64zK;N%$a zBrJJ5L5}`8z5qL)+{O#NTk+(cD(Xg0f6<97panpTSE0`!Lq*;`9#ZVmr@nKt6jz2l z=Y0u0+U43Zg&UWw5+jBuLB;)7yIFtDp|^)L$%xSEQQXu5DL#=bHAoO`GLLN+{m9GptXs6QPXQvr&d zG~BX8Tj;;o;m&j1`i3rj`XioOV<6>is9Dw%iJjnu)$7%2YxZS5_O$utGLrI*%WATG z-4?}Ek+#ah`zF9}Y}>Z9)^R20-};5;y|p(i`t!vWcF!2(*cx;Lwa;Pro!KAJVeZA? zQt@E6GUp3Z$e-$Clqj+8@4MH6Awg3VqZoQo#g=Z$Bx^DO9E-Q!!W?x5hkE+u|7LFR zkr<{wIu9#2s~5Z-aY4at_Q}r~Fg>>X@lfihua3=CX`r~Zinx|jG1UT&NJ#8njBWR%z0U(N9g989%d?R%?LL6(sF`jY&nKbu$1XAHTSlO}xi zwHjmh%P6rfkA-EdwvjhV@*8W_v|s!C^^>dX$jN( z?Yf&}82T4K*FTu;6QNsY`G4Vl$!%y@H#k-hyf1vn>zSM5hANv2KS0ZU&egJsn2gp2 zl+}b&CwJI}x>;AHxz3$ye-XeaepM#lZWRBU0B=?aaQys^jV;e&yEp4i@Q&YGV#Jfq%%2L+_ z(|N)B_dMh=^J3gZ=5qdFdXiX=WffS##%u*4@6fr$|-u%DKL`fHv zd2imL)mf~}wyBIfQu<5np_OrX@3^&ns7%8!3RsX@W;ELAKqVbhC|lrbdlW;L%+%BS z9wVj@m$#Ue$NNQ{K8`haUF=7${M#pVxX98yssY6YY)Anq+H`6S-!LY@w!X8dZe{#` z!;2tagg!)iohM}0A6okj+Ii1;gbeLhftg7=cINiGZTpsJ9*ypiZdjIlXV$`JHHUmM zO0w@N`x=IOfae-O293v(90TpU5L&~U#Cbu|?KbpX`p_VWThg5O5|!PsPY^Pdnacxn zTKKCBT92M;%YtK=rWp_A+jU@C%d_o3BI-{?mN(jsXPRKikVQ>0d)dZDTB!}Es~Y1t zDa8jUQ(vt`pAtzmUzVx=kRM1LP6Sl$DjvsTHnbF&=ft?RsYON631bt&sf{wksGwz7 z*&am9cuy!*El&~5ff0TRda|)J^6y_bxG1duv?89Cbd(ACfM%KN2M(_|6&*T zlC7rKRv;Z`d$>w6jXvPA1+;=o0*qPgIF`NudXw@0;%3M_x>608Zr32;;n}&Lnyx!y zJatrwt33aw)(fpehE5!4()n`L7?}=|u}25%C-Niz7up86yxH{h`Rk#1CM~jtC$V%_ zk}m2JQpY?`j!Y&-X@`>L=B;Ju36C-wk40HGpGlMOXtA9tiZva%)L~z|1GL?szB=O; zH3JFylJOGI5DQG1pev+u6pbP#fDlWhwMEZv6VY6Ig~5Kj<5=?2IC8oysu646ojzZi zqDi;MnKtK~{<2ygYgxPulV9w&F_>3iUQ?gN?M%7yGFHZ=M+Yohy!%cgp9oS0FLy1F z7x@-g!Y^@A>Z|Kfx4R^@$zF;bF4{33&GPAU*I!)XszIdkEYyUbp1>2>Wyw}I-3v=C zJ+wMgE%D2{EzS~7*VpZmNeu|J>afo=H#Dt3Tu?rab&diJvB`^m2^nTnL65Ls?fi$e z3NtTFWY7YNbiq)atgnMO3+@W_TsZarNzvsYLweUaYU5Y}2)*#sfObmn4|Jj4DPgUN zrGh<#&5t}hOu5Dp$VP0nHnF{I#VtUzohwVTgB7pTbHA`R&nio{DUXUPrOvvy(CTb7 z@m73ElR->gDSQP%a})Ze?@Zg+mYLNF(&vD)Iemq2H+0a@v426fNy4kIDM*{>6Ou4B zQ6!=?5u>1O605Wr4r2$s2}lug6~i`ah3QIWt&L`sA?Wdn4AfyWDj<#6eZuL9B-%)M zIBeu}y>QgM)#0V6K(fk%2Qq8t(*PTAWx+!o@6hf0>rwWhreP`>s|;eQ;IksFUod%^ z0#>(lNFEzKnB#grk0KLTz9o(ZF`07F14O@vQ-3BjqAIjRMM!^>WTN^rxyC8kQ zGq{$1{~h%RirOcHzFWnsZoYDftBA#55;a%j>A{s-ANKB*#OQz%Hh@m;%o^o) zZgIU!Hc{!~JDxsWnx~-rHcPXGV%no$Snegdz#a)BT%7+h-xhmsKFES>H7!LGr(iA1 zsf}%?>!+kLze1x0`rzLu*7Fc*-77FQP;X5qvY=nom~nstPY2WJpC;bZMy8`yauSBp zK@nNY<^m-!xZs^QY#&1`oqIxUM;^WqxXf3^2Ig&f9lOx|Y)}>E%Qh9g0@ZP>E0v3< z$Y ziPlI%TzvI4EodsnhhA(?qiP)v{SMD%zi#@rMzXQw0_mD*dkyiJg{T^~;IZc6Pz}DG z{{E$?M|TZf20I+;JGIkesjca(G8BUUxzmaAg^4th<=dK0 z!s#!wrMK)ov>!G8_3JmXf$4?SyMOLCPKv{ZuPbpz zxzbK^p$iBfdjGV=bMioJcqTXFVB|yyZECb7Pt|QUbn6nWBN+cjJr#Cs2_|qW&_bq3 zU2@`Je|JF9H~!%5=U!t80*I@O|iL|qL`S}eITXo%9#6{4uOP|qc^qYhM>yi z`lQV=Z%Jih(DSZN>!t@!Tg1>S?Y0;nzVR|%B;xx5ea3#Q97{*0jBAaUY~Ozr0E;$SqbUdkr>yb0LJ^GdTJ?Cb%MDAE{tvPC9C27h<^bX=_ewASiGS zQt}m#*G2& z_la1k(L3n{DTP>xz9#G-VGRxnG_03%|CE~uz~0%5DRYjpk|_^BxFR@pn2G7}1m@r& z6gm6%u0s6OT)Z5q4iGjRBJ9V-(mi80P1ZR<*ii5R_)WlrF4G~KnBMPLWoUz( zg`_j<%|3ZaSD<_n#AY$JT%O%?T>SZQgId2M{ymdLk?hJ&NM&6gK$Fpt1@QIrRo=B8 zYWrnJUgrNc%L*SLQw;Kabj?t@Pf$|OsZv@oL?o+DZXP4PjdAEj~I6UT`Eu3?S7s& z+Yo9mDPR~5zE~X!UolIC>a}B>*O3dLNvZ>R)o>H+v+FVbrRbQGi&GE5P%bYuxEh zO41pwQ|p^ZopD#f(=m)x3dwA3!|mBUa5*Ue?4MRI<@7#;$#Yc zIDQJvS1orIa3-_~SG1bFdxq=8@48hPO=1}657)Dh2QDWEFnkVB&B||9sN%dHy8-7i zIJ>haSd9MLYI|FY_HQSKROwuM9XM`FwP}Q!?ZkvC{ zRx9(9DgloPGk7eZ!q8AZvjIK-yCF%n{p`m8&#h8&*EjE*>-C+y*+LcP7`BfCq)~wc zfVv*d_yM7RTD!Tse3ahR&wwUvFw9D(L~66taz*|q&hDkw5zQ~ zzwCGEy}x8ZwThaM^f`E$Q1w+a6~w9kE`PO__o3YRWE(KBUuL<%1bwyO)GGuX9ld&m zyOezM6VJJUphQpx6ydsY)2@(M1Tou4%vwS zZ+juSwtnps2>L@94gld=lTLd-$^HBtDSA??m-`q%LR5%D^X~Vx3|8azo_@rSb*B6M zWFgpb0D@V0o`eddi69~)QvKJzBkt#V1@U`)>P7Di!`2@e-Kms2)1^%i{{$F>F@-Dk z{5jy&sI?L!Zi-*JI0MaJRm!{4EJ&;g7Q@HeKTJ2xE_!uQ?W+|aamYGYnUOy>`ZICt z#)t)R$uiRLe9S$k@&U&$DT_P8ypD2(7`|;!_nlBZ*qKF#A0<~ptMN~Mq8$dq+cWL4m;L zhow%vk~T}jR^cy2Dz%9vGe6bb#I54GA~{R9Z@*4OQl_uB`VHpK_aUA8i(X_~taixL z_B6UbSqj44e__l%5K(i8hm@y1?9b-Rk(nXkv5(`l5NrE>>Df-#D(x-Hg-w>Xas3Vk zSwFr#9bxb6E^H{*pBB69p1YEvFdM|Z70D(KPit{KcOX7TLzqy9S963#O(Vmy^P0T3 zcZfQ%JIb3)Pr3C)n=k2ZU2Q_nT~ksGnANe=&ZhmNqIT?f+r#+(#p zbh>AsHaQvcsj*Jgf&q-Txfq?@y%KC)Q{cg~ML>fmA0oeqFC1reDz@n zV=yiZZBgekzq5Z5E0tl%hHNEU$oTn7t za8^%dWMY{WmmF$_B}<23*0eumemuljtLug-*U7<~CKxsDfSOZXdbZ9~-NC zifu+{BL}?jPdZ+xOpIXoE|t5h$;C`lzfzM8RuYc&Ihe_-Tz1H{GJP1-*f*?`}^e7QoZL&N%VQ^*N?*t~G`y3I``#h? zxSC;yJHBXS%-SeR*iuS4;z_&;N+#nzscc?TYN5dWp>^ zFvI6GT8@N)u1=<)k>o9orGTUA4lzP{KHK+QOS9Um^m@)D#^f~@eu*ez#PuoNqnK`F zOqRai>}~VKdt&RF6!bJrC4K7TwF9fNCbTNIcT$d@Q9EFv?T{&J>*mxqvwdcF8;213 zGj*C;w7(qwYo?`u^0Y>?hrfaI-_PH}_+gaK26&^MvE?VTnq+1%rc)=j!RUBD|9@z!pV4Yk zK=J>?C0f1Uq^V_k=hJ4TykfBdzCHE+*;|pRCIaw0)!qXy z+lO^=q;|}NnX<3oa_eM1{(EB|pZs;5Bf+fQ?BwaeucI~6IuBX#{z1s;(qqh-Q=KbD z>Ujia3)G0~)Qpgpsgp^5!SN}!2Db&}mBr!rK)h71I2tf_X5n-9iaR#Gtd=rsSCH%`oZgL=H@Tj(mTZ@D{b5t zf`jqljr%Kqjblpru$x|`V<=L~TknT|j^F(ed^|JG>t;(v28;b^{6R>ULs6%^c2k{p<8 z+0_wKmRf4dO9ghV(L}%UI&hJj^&^5 z_4?XUauzNpc!X`_mu>KcZZRU;;vM{yU!nAt(Ux3Mp}9}JqLoXBNwZGaqYUgoN)(JR zP4Gd_zxb}*(1|>Se>7dFwH&)B9LgemBIdA?#pT2l@|PRaj$>is|2tnhZD(*|OOHAW z2J>|#vnI}j09%NXQ8;nS|2F-Mq6I{>$;-FY#9S9U12D;fxfrdi@$4thNS?t-@+abA zUCLF@JEy#K&85){Ij*oD4qk;>lDw27ME`U};O=9tbNh3RHaLV%VL2jwM?0lrQXNxt zxaIUEI**N3qmyqn4vAK&jvQJz=JJO_j5M`1I1*SDDYrSmD{5t%%+%7kyq zWG(!jIIJ7Ep&{qW_e-Mdu=E4d`ox&47YlN2Dntsb!<%!&l9-Gus7Jhb)U}asPpzjx zSur&=>9xmN!k%Uujzu(JzWG~UpS-WhWTxx?Lq+8=&28X2swY>FiOMmmIk7m{B;-!s+?gQXVzIfRiltv`7$nmOkW>*-ROni~lmW7!Sw zO3rIupl%KI7>z4D38SReD%BGJ!Yup)KBAa87G-XhfpB!19#=;ED3o%kf)uSZg&EKt zLIB@mIN}$f$$Ek0RDwnNkiypOqvGucUu$~xGYBT7%c3=e*9RsgtIl1?~7>c}8 z(QTUJW^yOC4>9l=axVPQ0OVJ~AFWgMPkYY7*QQr%6U?;6VE12SD_^)aRtzGD=_=mA zJ?NZotYOdcgZBif$WGx*zxYw99CAwAJcqzi@WVHWBd2PS8(vdw^-bvDmVS(~REV3_ ziI#WBvxdKdu}iJuRY}d#mIJmiGk>nTXNIzo)J?x%A`LjhCIo|t=*`3_i7E8vjDt+~ z*~00sfp$aDS^Y6K>upJ@thopJj^h;_&o=~_KFz3J($sfxDw(d&u* zk#1gx7BUE5)7beYM6}fjIc!hZl^vT#4=FFF1t-rO!l>(wMb?h142d}65i4e*7rt)k z*6}Y|UcnwP#yybh>@M$FJ=1pv--3!VhUI04m^k=hE3~QVAp;QM2y` z+U5N#53;z^-+Se9_5e0HSpUCj1dut7L-?aoW)^ zfTo$9=f=k@IIyou_7U z9*H0e{b7iL)V&TUEA_r*k}f5SZC-N_k35yu#YC(7T%n{(>CK_i{-P<&=bC^yOhh#D zlb8RRSHz{2lbF*^Mx^NW0x?ID+jp^Nr+&7}tPF^fGu`(mTuLrX9PA%=oaluOV6*&u#7P1X0h&iCk?^1n_uH!6GPN3xEA=>FadODsRj-5LTHi__5~Vqr+f zP_H*tO_FfFj1O~TruB`B5)x=B0xf%1{vR>EjnFXOn-+EoKZ`>Se5CSZgZ5$c8HR#` zGzL$oO3Yr@)eA8yFqh7BDSoArAIEDyRMob9^EXg<%9T-eNK~qfqyzU^&D`pD_$=Wf zYc?qrIfl5wb0i>0dp~Z?L0ISM#jof3!Wq8HCJUCpt-GI~?iuss^1N!lz6^g%lWAj+ z&8r^dQnoepv8DI*hL37weWf;|SX!?iUsF#!F*sWTyI5cm^;S;djbqGeThg^=jyJBd zhPmU4TB7gQzdYQ7yM82$xqeywX|_mE>mT82y(Rye8L0=1ZSU6B7RpP$FWe}NF&1UW ztW&hV%#+&qWAFksh#b9+b5c$Qc`Q}>&?bb`4ii6%>PIaeSV9o2N$T_qUOa!8v(z)n zUtr4&b8BG=*5qYu@G1IWgO2)h*ee{ji8)O|j3=C@tKG3P`$duybXsqu`My`Nzuln2 z!+Nnn;=tHvhiFU5vemx{i;_$aw9+1OmFa8h>zwCw`SC#RmsqI7>ZqO}_`CxNfc0ACqi3>4WnwbS@gqiAD3|DHQ7gMUa~AK#4$4lkFJ_%P zl#WRe3{|s7GaZniz1lYb2wTc|X{@SoOa?A5bD_xO;_uXpUHY~`kQW>bV(!y>wpieN zr~gII=e{wo;cRgeo#~b>{RJ~Ex&S}Qx1*6dFx&w@7`eD9ht+ly=shj#e?JwenC_HSsiEMx; zd(p#3KRd7nY>3dOS-xGFw5QeiA4a(A`kMecC$?Y3nonvOhf9X`#V9$P&LICh7vtry zw?dAAnUI8Kke2@O2yV?9AhD<8hr7aqvlpVv{|q<30bD6`QbPL&14lB?w(fPC<%f?B z3G}&;cSF6iBw=))E1ybG4WNz_zZL9!SjBG*-0r-~J=G~B|MOjUs3dx^BX4A1OcX+q zWBt_YxNCYf%QH?AGc(g!xY_IDy@#mgA0XR*5^U}jhKEyhBO^!?zGwURMybFT2BP5CM#!>w z-Y}2Mrgs*wGret%uX++W^wHM$Q2_Y-bjlnGweK#kE&YAD+?5hi0h!6#ga1=hR*2)R zFGoW*S0lt9A0Lj?{}#1Gg@gQQug`8YGLhR3?O4zY|2=Mo02XHDyg}KN}xvIy}|t{oPd`B#+lygFY|wQs0C_ z!=9g_PMb|ql=IQjnd$=%SyvudlK*sma{R--`8UC(t=4AEEaHAC`?}50TtjAm1ZT&X zHFFXN!=IT=hrcguMYG3?3a7kv)MOaj0}KX12A_0{xvaFWS^ynv2n3mu!HX@QuV7LB zl@W^GKRmEw*41$xoFaLrrg?SBfzTAlBUZFguYfVK&l}}?D@*40u12$dUP%X)V=z1l zejx=>d(i96KpXR=D2pa>D4wD&mO;DMFNH4kXPZo2F^6Z4^uET^CvrK;2)7MZ9b;dy z5}kI~Q0fZgd8XDC5@p@HQDMC39W3!T=M!?XgrA8%cyOR2X2n2sNVuvasjafY3FTXh(;FEdsvIlPg)gL z={~;AEAdBG%v!a26Q`3Mlc1oX-!t#ar=ecL^+4-6;ZX2|pa1KM^n_>nR>~oEJa-dE z4n?RczlASGccV;s?zBnNH+9MB94!pI_f$(T=&|18fYwIgy+a#TcPv&Ti}n0b7L5XW zTJVMV7Gj*TYqsib>lEj_%VnzVSEg>ub7eOkc%5)bsZ#t)e5QpQEVc8FfK|*@dP2Jt z*beUZenPDDfokI);jm^FrrI@#lTK=n4?cff!7I(RwK6NG%F3t`^r88L`<;^i5?mLm3>*)S2d1;_-ehi`=}DAMR+U%uk2))XPi6Qt9z$hTJy1PeUu?bp=(t3N7Cv%F@{$vc?n z-Dh8`srrsben<}SqPMBR!&+C@@}pqNrR=Ue>co89rZcJkoppE4p(On6+4#IW3E-{pCa8)QJSh-Q+~*kKlZqcby2 z#!21q0_Xkr7wHO*9=U0PCEJXj91NJ#eXI50>Ro2qW%g-rDWg^P$#nQznMa@A7RDQ6 z;5a9{2%)E3#p#loAG();hA?)F|45tHo!T;3ZFOpauD0teax=y?M|F;K?_-lY?PC0w ze0#xG_yB|b(hVgKG-e+BuDo`{$}4%Jul$83&W`I1x{$EGRwm)M*7NwpahA_wFq!4z zVF#Q;mG!=RzBzi2fyT2=`1wN&VO&Z;WjKX9wm;8A#V_GLA6g*NDTcDf6ylMMWICTP zFG=IAU$)4Xw`*JCX`GqHrjDJYCJdm}e+J|V51~?p*@aS-v&>iZlEJZmK>cGm>er(7 zAzh~f#3S>KL?ZS=eSKkxysXyjgp_ia_t=f9xA=l+G6RO&s^|q0p`#g%EnDorTVnX@ zbvzWRfQ^5XpzE|8`MMe?`U=XkZmEPd0>G(X!>3+n zFd&+|;xlbpKkx@xzsm){i`z{1w;QThTR_UK^_i_g;!9I)``#(|Ll`+$YJa~Y06Mk}}Pu|Fp+yvYWs@KG(6tUNM zf&#{w(-(nKd9IF@^n)&JFLK8BMSB!()uK?c?PO`pHCFz;44x&qr=K1jDn8_{=e{Tx zrsx50x9KNNjASs_`FGa+O-` z6hFJ?>yJl4K!bjQK-G8hj-4FJOucmcI~{I#`Ad^H(qJpDQlLK)-(;LGkz6eZOU~AK z+gM+s^&D>zwU`e>1}mgD6VRuR^Pk*`ezGBYrP6+$@LQlEJ-5v2i^-5F6@$-c^wWkQ zO&E%s+@LxXB=HNivvi*vw1mA)xpZMZhY?}t-=hBZLd|IbzvB<~pW^uYE(h!MkjWE4QZI_%c~H?F5^;+=x<-7~i7o_V<19nn+!jJkjv(0&XdFNSZLkvt1;Q z`2EI3mZ9WdrOYK>ix5(+8>=hE_}mH9OWGjW)X>W5dYNG6oGy>dUTFrxM(2;|WTqjM zn78{J3ZD4I@*sNyqy;uf-$CEViSapBa=-iesn>~yp2@9(H=25?&-ogm4TGK9>x%A1 zKE64>x-{gC*I`o8pTy)IPuW;eqWzb>wamhec^?XQ0?*j{2r}uvJEn?#7ukeWszqxoToLStOC>A~pIRcphE z6qi3Et^x0-e56-@=D2IPKJQ8@ono9~i^=B-YF=h_zZ#tdj`nxmk|9{()S}$gHZRX$ z7wvKLBa1viru$bS4o*1BsIejEN>+jC-X2R74EX|E=Lc11Z!ymbNZMM-#=@t~dfIN}fU@H*=Q*BbWm zuVmSnT=0?8wED?XgyhJg$=Y;o$yVt7GfwRe;z1@N9Hy*p{VuHdSP=3hCbDn8+ zsyaZJ4B)e;`YhmaAIovyVJc9={WkOV8&JWW#_LCf!$IYY&Ih*|l{IB)2c?sGGaQmUxh; z`)!*{(ipToBh`MDaIhG-mm&2RD(ym=!kLGTkK3x{cOH^c_SSESQudmA5Mp*hisG4+ z4ogiCIkJ*(Z7!uw+?VPzEpQ~+olfhD4%_*wN&mTl_V!OcI{QE`55{e*^*B^?+&J<@ z%61wwCShYnguOS&TQ9|^Ubil~JbO^9tpt5?BWCQ6$Dx~33PSaNE^EfqHyyRrW3E)F zb~$}oaMXXay4vOSwFLFq0~YG3u0KE3=vIea(@=XTf28X~o!9UD%a#Xz`D#?t-5Y$a zr1AW%(%sqmxl6?sH});Oq(8sB?z}7`jTcvrnb^W0$LwYBizybRF?f5@1V+kaW38D$ z-r7S`k>@Dc^+wp@635@Kxzn>WP>M=jlBD1wA%}$rxx`~Q>ybi zs5P-re>Qx1=XQEbC_$8dmb>0DaPnqFHbwd2<$tLCb@!?Ysgc@UN#Oj{+mtRA?S|5T zFT4F5x5L`+xlpco7ivH-BDO?hce{JWKRUZBN*DBiRsw5E=SBL}Y}0?u3z@!~;fZc% zSoA=j3Z=h@!>_u&NGtZ=otLV>-0(q{&tszdylQk}cY}vs0zbe}%3g`s{99hlcmKVt z$tCbx36ySSE&7?!0}tsyN<$s?N=Rc%6g0QOpnE7*Mc?;i8@X@Pt2F=_5;`tG|1ybOo8fX=b=@1QKP zI+gANG7<|X#Z)^Sy%t%}M4HQ{Qe~ZS#@H6BG zz(A;@8KhtlhOA~wz~5-;B^f6ZRB9@2F`$Q%uuZ4^V(e# z^v=&K@^oxY#0cz6*crM2U@6pLFN?f6wdM_0P{nBbV-}Pj2)sWAB0Tl=GJaM5jb*Kg zM%mo^XBdwOh`Kn`xL+Oe#_HGG(52~JdT#-KrTyFF8PVCz4eh=GizyP_nGh;YsAhHP zoa(1^r)21Z95{4=&v`^P($kQNZ~2J{6{NBZfzNE(_?5g-7laQiGT zym{te*mmhZo|_Bg%U%IL2uL^qyirGsgOZdW5e@8%+J#s>N%zUgP&ItFOEI%fO0W{# zB+sE6lM1eVL|x&xt7Ik~k0*D*SNON2cdk^Cx7G>YG?Xdpnwp9Ea-wh%^shW@+=W7L zJNjfF;-sD0TVO8StpJ!+#ti~KZ9fN%Kx68LK!=~}DzKb~=a=(>z|;O)L6qCZ+KxBV zU@ROR3)nl;4z8pQN!9!Njrn=G(uvwvult{1o%w4g~vG7iNt(0ZMOA+Lbj!Fkwkwz$1C3 zmintR^NhHjhdFG;@Rn??)^0x9Sy2ypbpk>huc?n1be#R{hdzS=Fkwd75@6)%c&-6o zzE*K8pow}80d#GFy2%Z`j^nUdKLCs1iF;M#_tL2FIG#$Iz4qMaA1r(uU^rRpDS`&L z!B8|8%jH{5l0@m$Es|!$^rPL@8v~(+#xM$p)!~){Z;Y`oQS%KO3({ z3a$ZnIP)2TTMj?HnuWV2M3J4lDoUb2@Ax~)hW6^zfDp!hONkwOqrqhMvgG!QL&qxY zVD;kS;y(B7?im^yJ>vvm8Wc1X_^1zem%qAu;_&URale09H}EtabboFS49{LyC6vG$ z$VDJrGD!cA-yQJW>nc)wkS4ZW5kSFRCAts)aS)R=BQQJE$mGZ~-58tp zGDa~U2A<9vGU=AP`9c5i`5V9xU zC(N97;cz#%UB|ku*&&Jot7r&WIRKO(BjSV$({2~<_(u@;cD&}=2^tS3KvWhSbdAp8 zWTZCc;>qBP(D+?941dBPHJ>Y6+bl61?)D$8cA=5dB;si{hrxwwyAt z9(w?>)RBmNi4<^tw)4a5N|OFPCc99biGiBeDS-MQ6v3R^@Xz=|Ai4(yIqKs+@PB_^2aMZ`LP;&Gf(PYKXFU^>25u2j7`f?GXEkJw{mx4Xo3j z|1dliY`X;MS2xu0E%F?=bl&?Uf_l<|iHy^*X+V%W@&6bT(JjoB-BGceKhgXiIDdN| zjml(plJx;VXX%RH{V5Gl=Z9j1TPw1hoK+sa^N0qjsIjdLL_knEG1496*+gg5rma*9 z*KaTzeo!=KRA;`*tFrUL0y9e$z&aj5FMKME!7h`HWoc)9Z!;e~vT%3>5JC&U8Y+BK z=|Y%xd(Xoioj5G1SR)Uy_RCDyj9YK;>qh)z*v#$wT>m-vhhM|rpx>kp`_B+l*(SRV zhCk5U=DTL#!I!-J^NDCl!EQ7^BhT~j2Md3ORgi8Pl7fi6O8sTvN;|{j z9K~Xop+yBwbn?HZu6k^4G&)=?32KC<5Ls>woCyyT*T`2ah>_GWo(k>r*l`AjFu_s) zO4QM!?hKrd8n`~FGjq*|>^SG`u)oJL?O^XVfm=~A`NqG!1NKV$Neg40M2I1Oq7X?f zhXsAiZWyz_Ll2rOD!3_m(f5*EE#1EVIIfn>WV8yz=crbQINK2LxuXw{nhX3l9kM9g z_g)r`JTQ~VfcE_cXVx51_k^a5wyAoiNWa_-xno5h=E9<)UFEYN-N>|yrPQcZQ4rvc z3dA5UZoeqaczmLttFy^2nEXo+4S3r`caTIoWtJ}#!2)KBGCVSUqtCnsIX&el5|7T* zpHg(8J6E)q1xq#%z6MwyHS#=jdae3zmkPwMD3eQxFrBHAL8D-*sbl;a#BO@2Gl!Wq zVjRr<#|66pdruHeL{ESVQ+F8}5i9ijoBoBC7FjS481={_Lb4Mox(TWL4Q3<6i)jW>CW+ z;&ebCcn`<3I#px%m`B0Vo3A3Gq7>}xloOMpBS4h;mS^*vlT{j{Ajg&?%}RX7aS|hs z0^gXKo~v>($CjKW&VEt@-_#LOBdDq$n;A6+@1QcAo#dbm9EdyYy=@dTC&w@5VdsrC zqF?^Ve61-)hTu|!8wZwY&^l-Pr&Z2;IsC??9*xIPG9oF&nVH5wY!aJq4oE2Oq+0oY z$ls{RS-SS!u?w)@n3cCHok1wF+XuJ#apS|_?_?9?bi-YtOT3)KPQ-!F+i)ih=;^%l z&FY#W2B%#_y8=9`qQ+F%+dS@K5vldx3FmGB3hXOkfkVSCb4G&u#krn-gs8D4o_B z9n9Q|^Y@h3d0MTB2ZQwq+4k*^oBt;j&_P2JaDHIwyXkqS`w~VQYi+9>UV>TAi zKA6Q$8OylP$t#=z+Y>nP4f<$NkTYV&d=jypWW1{{1cB3EJ!zH~5oF`PB{tu#wiwW4 zJ_qQae)-kqvvPcE!Rw2Q%1O0s?w16gwwKb-jJB6z?t%4Id4wu;tTxr3@evFJOqRlV z1%7CK%PZ~gonJX=7ZB8y5!pHj3N&xcmbA)MB-Nz^CBwpbptp7e*SG`O40s&t`Z!ck zkODoShY|wqAv57&y0UmB_ja-4t<}G3;6hnCVIVd)4c+wNmxzvZxj}uBnG^(9475yA z#Mxf;m@=w!K-`-~Hrn;&yt}X1`U}`~Koy1UR)#r)A@fyaB0?XKn#9&6f$1<)MwKqN zS2=goK}K3{!9zonq4gfDFlEH1O~y%`Tmg5cB>NeM|0N_Q#JOlFSW%TA#eW{2|34fs zE$V;s5V)DTas-}A?SQvO=I=NP-G3km@^FLw;=h@BGBkjOVz;iu0e*cT_^cY9dRyq>L0C2+zm2j-=$ykGjY#I?A#FKWiUlF*sDh6V@8PG zP*B3>{A>Zr7@^!&K;s|14R`Ed3onY|nuBC9Do*7+|4*usN~-SehLYl)9Opmc+Elz| z&2ykAY=>bejirwC(-6d;>450Ipub*iwEHi+o$Q51F~GT8LAZ!{L%{ZH)()l1sE+@; zzV5o^$fG)D*Y$O7erHi=F!2L~8=|6x<3&{TjD5ln(H|#7j%#VjE?m=Kct{n0EA;ba z9zd$GDwWF|Bp&qoms0hP4xEt!GeUh_jmeXwVKLawBgbA(jnt*x06(TgkytH|-blR8 za{ia$r1!FixICMLge>Y#E^WW+-;+ir$lBdm4%aAVPweG@b|&4{q4GzD>3^rQv|RCA zGhC|MMP-l(IiA~Hbp?>1d)j~_87O<2%&Ltp?>w8{X_TN-!cxBu{xy+|ls41m(XjK3FP`ssOQ zI}G4dHJ?ZN~D1yA1|aa^kURaSn3F80!d^Ci0&#Biu&R3Kjtdj5q$=-vu&^GNO8$2)fzDT z^B;lh4vOIc4B$!vm6{0WG2cO3?528vrK+(j<^$1md){wI{$rBDPM%Z8yP$N9np1a9 zmInMMou+%~!B47slBF~P@)K%0?sV}6|9^$}hE(JY5m1bb{RC0cpkp9>61?E{H0B&P7u<1vkIdBN8Yt9J(apajY39rB zX4KnZLd2@^4`9(LQmub=rV-PX9g3`_pa|(O+afOQs!?2<(UqOq)>9?;A@YEvVCPT6pA)6eK=q;rkQr9|Z(*+qlC|a*q4=(}iSNzW@pi#n<26|F`!zGv(m&K`bOHKnflACoY0Iy!Hyes=i(2 z_5q3D{uCh0XXgCdmk1)wHVu4I<+Az;5>?5oa<{e{(ZKGsykt`8pQa|ZB3`gpt|A%I zuC$YtaXa7vu13w8wE+z52WC$l2K@OMxWbzHrMf_jS*{|V1;sGJuA3%$!!qKY0|Q;CNpmc zqN2}U12d~83zW2U^#7h7o1-YXVy8A2WN=QgapKcxa`e(hlnglPujK%s12kI5Rt7bK zXAUIVQzd{u5ZGl$JU7L+1(R+S{awdxRH4pA?x6{HhW;S@JY>*kD5)2Sh=HTk^=0Jn zYLJr~2%B30Q*D4=%}$CX=oC^01`pN zDYsNP4mOS(oTubJDp5tE<3K{Ic(|IOIs3odT)T~fW1Oq&@R$W~J}Yf5et^uxM@j8Qn6tU`mUqBF(-jfdaMTJ>r=@^SbQm+eDj@LUj$!-fvB}~0j$tkC;5X=^`lUd{>o1`v5fX_Zke7IOIhxVlTWS< zgz;~*s!v#U>EgD~j}f&v*DUt&eOkPedf|@}(my|FB>wPPoBFG!oVO_!Ptsk9yMmWS zta33Aj1=;S%OuyV_cKd>XWJi}b*{Ui5ZYO`tqM(6J|k*}GZ@VHCw_`5?Ej$AHh^Nt zS|Ny$zveQr%BjIIg#N-gZJO|-=OPVVYt%8ax%1$?^d(lGsPS>$aZLn=tT}`%6w{}+ zD(E>?t8YA*Nt|P=xcrzidvaL;(Ag#naMc@$7h z{vc^G4*0Iey{qR!Ve9d{``px1^$d|747(Tjo-9iw2ukg^W+j00&!)11_kVw@te|b|l5>q^p$YS%z zzE;RRvrCU0xH&}|E;6a7ReI?UsYDYcu7_wO9;VTfYNc*O=7y{Y44+^9{krAF%lIwu z!L8-)lM^|$?1UItxtpGIQ?5uNtZ)6zl4Yd+`ylDH7|B_MUc= ze4A|_i_noi*KXtbV! zgfFG5Pew)-1QjY-qN(ur_3q+_A)U^3cn8)lWr_0Xx^F~kvM+7IN`_PIF@XwPZ;mc@ z&qxL-(_9~74wep`U1=_Dli6It2g<~NjFCZl39~uMzctyk`8H7VYqq+b9UU-)GI|Zn zOkZgh?pBFQ7uK8V=2^%PfKT8I@z{q2(K7_9Q*T z7BV8z<>h$o{gIhqf=f28yUcpZ*6#*n%ibvyO|M4yliHz0pZL-2>Slnk-H(pXbm=N> zR!)yL-_(T_Y=FaT%SGZZqU9rL*yj``vGE7$MV1U$HBnzH4`mP zEC`)>V$5cBW6`3gkTkTvBJLt2vWV@Jo)-6-5CN~!vLfk`mRKVCe9f;2e!rZ#HF{X* zqzSOmNip(_DUshZBB%<&l_=LqzmuW|oSIRjD-lX<_;by;WrUE|@q{mEYY%vFT}ZY@ zGD0)hnYo0Qwm!ll>kiAwmH}?zF^eKi)*~}C(VaJ5q-Pvo5THuzehQGr6(TX ze7daDrMhIdxvEmkY$-whQ-JpLKl{O*s zOmfH+DBCp)Cw3}f@Fl95Wy2ty>AXdpom3;-EtH+<$m^+S(wPVOl=>T+n>EGftagc@ zF0dk(9{b(k{8TaNC(p!A{&i&n#q9Q?t~q{Qj8E`D#uFhPoP8CdZ2-x@Z=sI`Yn5j` zgb014-sgbVn~-rGqoZqfvePz~gcnO;5rhr%=4hCA`K9B9*5pnv((pH-W5YLf{F?Kx zC0^3l_cCN@;8%J1^{kbL?$<7+n}FmpZotdN1z*}sP`;HJ5Ha;m6wR5>Nc@AMBWh%R z7lrpcd2&xK!?CZDkky$NXDub2Zl3mi#HA*fiBBmDK@JMz?U&~Ktm3ox{5hcb!Z*g) z*~Wd$BOpdgp|}87YOK2SNZ)|0z~b)sYt3Q81Vm&0SDSX^u(@3K-bhFOThCQSW~Qfg z*EX$X{%49_&bu!j*qH4+u)Lhx7VL=ah|$&{u}d$m+Si+FCK>7>W<;6zx=1XcVJG+e z$ZZz499jr9A&{c!q`TeGo=(jN*e79=MwhSxhWUpQkQ2d~4`HQaYbIRJ@8{ax!jT8u zTm6NlgBALki5YMSL>-<~+#u7lmHtv~;j^ZzLo4|dp?b!B5w)*&o)9~+_-YWp<~7BdB9n)it2`{_qVPr*lEV5pK$@U-n4?Xu6)%x$8xNpb^on8@DAk; zBaP--F3bJ6t#@7|1LtG}SC})i*@^s7fs@IR?^J_pjg3704Mk>?WiqXKS%U*zQF^(Oy@cF5*wS_qq`_i;04Ibjf9y4FWMo9b7sq1nF`n66fXWkD?ypE6 zW2%c27vD8+Y=KV#z70cN-ZNqc!XocFp)Ii!j(cV248R%gswgz`plY7`<-R$@=CI87 zV=T2dtN$7p9jO(o^laT!MTN)p>%Dxe2|Y-|pj z*cjO+u*}zaKgPlP-Fzf2qr_<3wO%v)ICKlcaDOlXynq zz2?O1&P>Tj&vUK6B)BV>n->f0njQk_%c!?tWF-$VBu>yNau*T@a#7CeQ3X z7%z(#<-659##_r*447D-)^VEk-##*e*?yt}K3(NQOZE$u4(ppmFh!2C*ds&Mxw#lb(7^`1@5$p zmx!}VYJ^Jr@h<;DA>~Xq&g0AxA_~5>Hhnq73R|@c0+X0KcZzf|(epJJhITnY*;xfm z-41XPt*~mDH%^N5CQhW&m(0uT8xS#h4DQ-kX*savf*_Pg&gidT(*%(&;f<0DE_z6m zprC6DClg`3T-m)OdDbe`GGMah*rbYwEmeS|hDv_Lnn%VR*qY%oplJM#za!#k$yPlE z)~g3DbA69u2!xo;GWe$~nC$qJFCD}Y7xtF+qsB~)XySW4p{xgsmZ&eJAI(BtA56bG zIj{AP#%SAbHBNN9*;xrO!d9!(+T>))gw_#Ymi0~yLu_n`|EW-A^wNSRztc3k?pmDv zvN&A6$`@ZW8}S{Z8Xvc82U~fzC9ENP)258yG{#P&+)z9sCT)0QwKpFZK5p)+1PCdE zQq0uLVW|>;N+=eHf$TSQJECFzjAUIL4RaUR*6+uN3VV4aZ;^uS#26aiBJ=AM|MnKJZ`Urvjf{5Zq$pO-4z&RLvOS0;&Qg5Ck z8_|88wZr)a`d)jvoKQy`Tx?NzB9|^^56y`Bu~}J}cTm}BTcMzz9IdW95A}S-UOPvB zP}t>Y8mzcfJ#;Tjak5~_H9 z4~ec3v;`6eba9Uy(O1v2_pknbPCGSvqP3Q!BZLSvNPn{ zTNy2MG7H{0{7vSPSr}4a5$!lrxPd`uS*$;8pY3S8J2@=nPnsv1qB&7SB|LA+)`(rO z{77Vb4b91mw4}))R6iG`f_RuB(4OoCM+$zAc51)b#?OYds#~RHUz=z-Dd)Dii1O=j zn5K&f^O!$(s$~t2BDW>lPk$uWTn;un6?FJ?A=bI}M_{(Ppm3J?O+WS-?bYM3EmHYL z`sS4gpM*UU`pp96qHnBBzph4HX>2G(U_W#Ah`m9$&-p3rUor4cMt}p-=zKas5+?dc zKdrZrFqB7bDm~~;x~ur?I(CW2r^cJ{^^ZuKmD!JrkDKYq=O1^~w3mf5X}Zq>?o$8p zaL-U83*6=?{Vc-)eW~`bB!U9TwltC&@X_%!n1||`*7mZgS(g)OrWm^DupsxkfDLX2 z9Z-ux3^><*ta~v;qH~A8?YZ<#g8@@Lc3ufBWRQx@-h)ovdPN+a_!d?!k@2nX;C^0? zGObGkN#SB>>mk&Mfqmi9Ye0zJ$zL;#h^!Q?D!X%|-Y(|z=6sS2?4=tCOZpa8W~Lyq zHl7fQT;~YB6|p(XK5M<+BcIj(p#Uz$*E+D3yGT0$r!6bLcrF5AL%rT{pld%P~>h$AMRC=KEM86XL{YZ~o4LJKu>%c?t`augL1O!FhB& zKIfP=Zv^5KY zd$T*!bwop=N&TjxkGT92N3%92Vn~sug&?*wpY{_OM4(4|}v6MR|*>t-fMLdr4 zkng>6a{tsLY2vp$)q@>Nkp*P}ds?KNCiOLr5un1-c2DT!9dx1rz|nMduFbTWi9TH4 zm$9*EtMye#0NZHD4$>;;gV?B_qqnu$L5)X3`v?|hO@TFkNAJHPfE33p^^MELN|!L? z+Tuk0^~{HQoy&Fy5@VbZKavrK3$okS--q%L-%RS($00J`7BcuUTUwD;t)U+PnSB|^3dn_v9{UL zS^0#OFfJ>lzCyMz?xeW0oV6w~n|DW6yJDQyJjm>>{(5f*jD~gg?*aOl2vFkuHYS)` zPpwy)mJ4NgADTS|cErUFU8Tzs)kls{T01r=lDq@F>4 zgvhVo`BO!|KK)EB`yz{#Az;HqzB`^II*KX=u!Y@G+cdgKzem<%Hbr?D!hVBO)LIjC zb_EbrdzLfGKXifurX!X}GPjwni?3zNanp0Zcl511%I^^Ai7hp*J42f-&e$F>{^`|e4N9cXs9S(^!geW0jQ;>Q{V#6#)+>a7D_ z&cUb4~^)1?eG@%#XWI$?>f|8)-g> z%)E_AK`=-W`Lb1kC;ZAw(@>V1DTEWMt`{&?$6js1q36fImw{qpdW+15a;{(DG#A}! zA|&QEgPzOrh)Jm3f!gDkF-ijy{!MA~4Rsy^)o`wjG*)x+eEEmj)vZ|thK(&51qPY* z03p)fP4g?Yw-%G#_AO>fZxwIIUj~VdLS6!}7&##_8U%Fuubqy|ea#lG{nmy3t*^EW z_qrOAGCK~>V}bG-i05b*nQpJ>IC3HMcgz$JCPLS(iknZ5ix6kypy^jt3_>sXr!hqZ zln@eG%5#nOEV-Lq(~DS|++r;=h=?wtH{1u0=$;BOB<*VbfmaYyH1onVEUl2nYv zn!^L-TSE$DwG0I}0wLPvjmNrLlV~q-t zX~q%5-+TmLLyZpfUdcWju^OZX^FLMATG^Iyoc=1>?PHJc)fgM!^Sv`Hc~^~>t$DGP z!QChs$3lYSjcJ+Z->cip61!O3334*769USu?osT;e?tPAkTVfe7TchrgsDBX2TX_e(cr}>_@I2U0^oGvDz-4@N?;^PGoQ(H^* z`$8^9e6<~$m@p30|1oVRB3$J*RYPFQyC|Hfb-%^xR3A&?m@LOEfU@;F?4^dcwFs7U zS(f7Sh-w#6`JY*B&B77k;GMn9GkQb}>WviL!GV}Hp!??hL&jvwi2cmOAxjt$POpuh z&brvFBh<+6vwBdj3A-b};9)6_+iP5~1n67dshltCq$J;tDZ%WZ%6z| zW_)CMg!Z{$&uX8jTcPd8flYZ2JdvGUSR?vAvxAc>gI3^R@5?OFrfPB?h@lgIy}DR% z7b}r2v&y)Vq0)U*TdxSX!nrcqlLQmYK~8ejNBr_VH;}5%`$mWBXBGbaO4+bMspjUF zai698MA?s1@etE;6vJ^&*C(od8BKzHavb=PXIRvGD3VL`hzUh=|aHpCXmcPHiwDaa`$~tB*vPsEweyEwAW)yr1BFPiM6$%dt^$ z-E&(aLdyHTgITcf>%C&Ja_3}9nnRPoC0A08f{bemxjK=xK`A9@QSb{~_53uk1`(e4 z9(N&zT*(!7Oyc=tp9k-MOx*m9`_LXoGXmOqiG$SLIFNo;Q!xzmnHSvHO-+P_nfKNA zNyiyj1{;E(X4VNs7;tbihb2DmIn-Hgm-itlDDQM|Al~dmT;_Vo2t1i~W!RRT#YlDN zvM-6$yfv&WypP|PWDx$q(nA$Q;C$I*?(@ghhUhs(hI|S8oj8I<@9Ui=vF4-wyRgfW zcl>No11o%UqiOsE+C^_+t3d{jRv}{Lc<^rJ7?hQtLOZeOzxVVM=Yf3W;HEAxOZ54H z0ENi0o`RDD!-BBQkp>L+F!8}Y>qzhJ4W%*|ubY)y|Nm?6%HyG4+rCccsOOyI>FAtf z$#dGYSfVT~sG~zgBqb5!5G5r0ZnSYa$dPO*OIedGOSYL&LJ=y<3}Q%Q8%%~VW`>#f z`u)ZX&-3T=dH;Fe&$<2x{kD6#?(4qp>;8W4t8BuuZEJ3D{)dQWZYb4l-_@08a(u2I zUv~sgs!nsslVJEc=B;Ng7&VD)q1Q`pOZp>Xmo*JpqjySQI2tL7bv+)UoUOZJEzZBd za6uA^;D8RG)f08L7p?z#reyg23~XuV9@pI~WAQ<=(rr-1vFGk!+fZ{urb|y0O(SLP zUi(3t$1L?^8tgOR-9B40)SSshYB<`$U{cvStnU0t)Y${a)O?mnq#WNeMWW{8g1b0+ zwG&;#)fwck(zLD?%d5sQHrsTFN{x{{L|FZ@KgwBkq?D{KJdKVBhRg9hiPA4UE&F%l zJZ{~o+aIx^RmIRsmKpWn0I?%CjqUGRIaOt_-IZA96K_JOg&LVz&(x0Gg#FSB00cDy z+G5FHx`BQ#-GhUdN^7ir)|>e2O^9jX?5pgN$PSvt`=I9k{GjU6OCXV9w#oRd6#_9 zP}lv?_DxlsmW6@wIk; z!v%~#^NWuKTLmF$BJrHKui9T-~L+}bB=Z|myfs>)z@ zJ=3~?NVwQ>DkTIway0OMd0RYC*>aP^!~cnS;YbH}ByxMyl<${l{Wc7@uxv4o|f9!#)shriDuWIHYXJc&P?T@mh=! zDI7K1uw54!5q!x!-?kM$68%A5fcgCs=vZFYKfdAzec>i63E)KZefD-T1;2Wc)WfJd zwv5ttyzfgT5Qz!^DT9KT*FQD>oEyLq$GU!)LU5V_>_|=eQ)X45JyoBrrFv5I(JGOU zBM5LvGW<$5EloaH2qm_$P#_BE!+I!#DqH(@LFF;Wm%>)d|HC?ABsOQZsXgEl;3#a6 z_v|ne8#^x~52rf0W8VQT^WJi4|Nf-()&XEyP>sVI5R6_;Hd^*OOU%s5%mdezF;oOC#fQhJTnE)cB_#jP8f)N@ zIxS-q7HK;EF07BzQzL-Y@F{P5eq1`;-Sec^j~IqI(zGbJ5w0y!3D3{e&j6*sON|7& zSc}pSVODc=TY78G)hkvhjc^r(s$Ev3DLEj(HNIG#7N0=WPQXXe4e%C1y;BXw7F1Xy z1XZ}Wb(>j}{M`w`qm`T4cHo&fdjjZ&kEsTj$Dr-?r7xpeU5>!f-diY~845@CKltJq zi3jalgpkwMYcFHg#@>OuBKWS(>Iup}0@LuPSo2$wg;ln(*vF`9*C=Z-Wf=|w42i!; zb-%2ve4^9pKQTQ`_ z%5NvC!UZby{x?QVGJ@yhgWh8^W$VUOMHfGMc*EJ+fE$PPU2!wHeNfvVEdl{mom-Fzo30C!s?-v<)Ak*E1Q$esiiy=Al zglEcn^{|a2xMk0TA%L0|*VN&u)?UA>8V}^AfF_|5;+VMsUXD_yZMOP-gTV{&nO4yn zW%(bs2+WDa+whp>>}4q}RRQGAY&(1Un;+T17Htae{Syr9vxN_N`W&8oN zF<^2FI!UDMYQ^!uSAqt!QBYVm!WqrasM6L@nmdVLl?76BSiM|yv3eyx{t$E1Te515 zt7GX&o{n00B)l=^DImjVhjDnUtd>fJ@%B|L1k=TDv)vuf7>MhYNzdy=lGt4Wl&JnWdK+&jiTxu zi3Q7Wm*-XkC5xZ5aDA?lQKxRe&q@N#8c{{*N&JP;$(Ojmmd(rHdF1wPA^a}YT;XVE z))d8V8LIHkN#qm-l+IXZ`}^}s&7H`R@Tk;S{L+r#ERklsjf}WsFp(%Ok$TUnR{;P_ zECw;e4nLB+z2^qP33`WE?U3aqWZ+$EtMUPB4G&*i2#Bo^={BW7AcBtw2ds#EW*=}) z7LT4c54x_ZZ20|jo*1<=&(d?gJ?6n7aAO#+$3HzDtX1!%7y0#R%<;(z zLm=@F0q6g872|PmzBL4)_`KT(Kkqz$&>vU(kwCgO^VnlbP7y`)kp_*^vxbHxL_|0c zlrtT@BR}*}`SH#&GkIAL08zY!D2Iq-vcC@-X|T|&zKm{h37l$&mRzS) zi`?!0C7RlX#0GgfW@nn4wYmu4np9u@T4*p0(cf=GB48J_3vq>>n9%aIK1jp_*tz=_ z_)9~p{*GeaCHgIs)UWzx_5!2LLX(}nrNhMZFpfj+sY*#`&aYhCe<|6B?0q7f^r7Te z-OZbyr&+BF-@WC7sN0jtXK{JY*4Ub_laNra%yzr|!-g9l9(fI>96s&!-B#!B{9`JG zr%IkgeBV~`ZcmVC%MX{_`ylnea9_a4^e5)@@#J?NG!LG3E2rtx!vOs7HzMx_{h6BO zQ1gTa#$?_7Zvxq1U<+>{c)~qN^~XOq-wBxPx-DIR|GCfC4ra2E7d=+ry9Q8ovy5^5 z{0CbYlT&Vhi`?^w+8eH@<)w_3O38v;0H;vq(UqJdst|_panrg+;kFT-3b68l!$Wyr zdue^05bNv@vkfB!&~P$FMw4RVpRhbfeet{DRNzho=leuq{BQxBSuUrb2ZkH5>Q+^d z9lZv{cYB}{sll3wjQ(U1Ig>GjGimt+G$KUN1HotxQ*Hu)wdQpEFYfuG}k z`Lb1DP-==Qy{MhN|HRzuLrB)AoB&vvdV2rkgwW%HTguYM#U-+|4t^&RQi03~3ER|8 z3`NG+mkWFpk-H}25;3coGYGdB-BWv-q1U*>H8o;7Qd3~4Ys}K|Dx3vtD<8-`=F3emu=-G%J^w2j8 z!DJ3hu6%KhyuS3ws$OxWowu2SYb2>4TL`S96z%Zz=(hVrG*o?QptKAM86<73;bZS6(w7cH?^!8e`H zj>*R`;~^X3PH%X|;g9P$`cIBRzqOx439@NqiXzBw6z^R8TJ)`IVb&5QQ*j5UCfzZ} z4*GD=9Wa<1>7j4e5B1PMM;4{*smu(Z)-usRhlTcIbJh$_bD^=BdMOS&gU=P-xAtro znKED?-W!x6?%t~gbGV&S@08&sDAk*k*YVuQg1LQoaok+MrMMvOLdsgf`>$nd&rzb1 z6S-OP-0^+c=kS|XAk8!Z#`8>L!}XB>p-wd@-Db{!+q0-plqYvKo(EqRl@LWxACRQ7W4#;2Q{07ABhvKn3O^iitv?7 zYCu@JvaPdWZ#l0=;OLDS8N{;`io|RAX}<%4*2*?hKbpC&exHy(T=-ow0!n{?Hsx>M ze|4-8I5IW{Yn;LAplqCX_jH1k@t-{d9ZLk4`$-wxJuFY#znm6}b4pR1N*P#|=eR6Y zIUAu_@C&367j-CeX$1?6k5Z!`+cmHlsF8_>?tAxFGCp1>R@rZ8LT*WC6dJioIT%A? zNT(=r20>@W!O$BpXhS_WChBcy!pO@DQT8HtJMt`F7}fe$^-g+Lu9-pi>FmIy*`!rR zpI#~SXBv#;pw~K)6wHT9>A$zIN>*tJw5Ra9AV__z0~NA@A{3tL+mo|2g6!f&pLe1b z#TS|X!e56rk}fAoDQUr}A$$o@+yR7X;W!R|U4Psd7HE92$wkigFh_l?D}J>XyLK#l zx8fVl%P%Xfq>=NgERgO~_)&TrlBJ3-Bvpju5XT>x-%l(voo9EJqnc-cTAO2=M)rPQ z(#dt4T2DsgIv5OjF<1!-!d;FELcXK-`dr)lcEi zzkrH8jfD}oL_Su0yD~7x*zh4b%sf!@nkQqgpl(#}rw97S7n|JEa$rsQM%$LP!%k1Q zOti_J0m_RFn(_SSCTaOdAz`{EE`3G5DM50rYd`P=);QiTYBx_L@bcod z{T#Xoe)Fb+At{2|H)S#>%!gBpoLO0jt`|ER`>prWNu^0dK@CM@!ePtg`^k)8m8xDN zfaZmCp{Du7OWYyQ`bDvf7QglgJQ7&bEgFu9UBY?sw{(Pnh+2(=ZE{u$OD!Q{xsLN1 z@z@JRksfepx)~s)ASfc~Pm=O>M&wQ&Ov`+2iIduaoKwOV(t~z+Ju;}l5IQfG1->rY zCtPV+ZUbN{e?*?4N?9q0&g~Pru(V4j9GXgvCz{x;J{JX=MNe?!ic-5(1j~ z$`j$`rJ~H|;&hWb@2cYX`%9Kf)0>jo$$pJ^U^9#(< z@cx54QHs^sE6c${#~UE4v3DqwXR}|bvQ{kZQvM-thr|t610mMJmzrPvC%Xv`6YmCL zKE}-50@gHt9xhH7ZFGNKq{#~s$8+9Q59e#iSyy<$pb?g>x9^`r3xtId1I{FurKp!6!3n_g~Qwk1qIzRm)=yMwL)+I8hs4%V++haUG zv-i@~a)r`aSI#^~N4Ap=Zot53y!OV<=kQ;NuJb{2<5;e)c~dvRWq$IrPkEdgwx^V* z6EFtNC_!T%L#|A8^HTCyJ(XtX>q{F0^Un@>YiL5qO8}lQFv}n@VU91PiJ8U+M+nQoY9JXsPNlszwU=7CzS23g0`7V%99&?(< z&-;v~NtQb}neTySG7lRY+|g>hkfO7lfq79d<3}umY|zT^z+yUOcqj@f2yX{f2a{=F zkwyCyItFcsI+!+Ulnna^EyF%)GV(bqyv9Y35`(kU<2n^AJ=WM!8KWGfp-4PInhb8HSLiOp)aXUC8FOFgDFvf5EuNfjIxm81E1ixdBPx zT}&wHMIP^^X7e0-#vkY$T}a8|cJL(O2B+EhvV-0-V}t-%hK@^;)pS@kZ@Qi5)`7jR zy_=e~ZRWv%V?dMaz|!tLc8oSX z5iiD?nyxnSnWT96?S0}*LR!N=o-^t6*W^JOp5{FAFce=)clec^OrMBSiyI!kNAq|} zr)ASfRh_#D!zGbBIm)hju>=#cYi#Gq>B_raaAuJb<()ko49`s=!OrGah}MGOQPdWvNmy~ z8aa21FL;mnB&6tLo^VOza}`lL2_N29nljH<;c^IW6%+X`GYc;5ogYbLyXuWGSA8i| zpGT$>%tz39i(04FqX1n^_F5Y>oCmcg!oxZ6ldNhryvOqsUy;dPE$~Zx&nZN)!&*Gf zB0p1f^6{hi!O`waws`T2EN-jo6KeIZr+7grIXnsj6MTuHJ`@P*i_X@U9iEx^Jp08b z`xoC6gWeT=?Oyd#g`6Qqo=yqxBS|c3XJ)WGa#K1t0!6 zWM7d@sW7gHZhv7!o~7F*yWji;Gu%9%H~hduBExAxqxOE1vmLcG{}Qymcp?Na}C zwVM&k>aV2HT=yy#aEBFhxmnFHCa@7u1Ilx-ABa_xV;h3$z&k=Un_AIPG}!c%=Qs_S z;N_5f_Z83BDP@XqHB8$S#*?cB+)Xdmg?|E)=`X-{a{7w+*8Mh&-u@M3#tXG|8Qg)- zr+L@haVlag`g#1to>imix9#7*iyMg_90o4mrEqF4qPUOxBOdJ1YUtg q{+8YG_bZkf ( + Tuple[ + Annotated[DataLoader, "train_dataloader"], + Annotated[DataLoader, "validation_dataloader"], + ] +): + """Data splitter step. + + This is an example of a data splitter step that splits the dataset into + training and dev subsets to be used for model training and evaluation. It + takes in a dataset as an step input artifact and returns the training and + dev subsets as two separate step output artifacts. + + Data splitter steps should have a deterministic behavior, i.e. they should + use a fixed random seed and always return the same split when called with + the same input dataset. This is to ensure reproducibility of your pipeline + runs. + + This step is parameterized using the `DataSplitterStepParameters` class, + which allows you to configure the step independently of the step code, + before running it in a pipeline. In this example, the step can be configured + to use a different random seed, change the split ratio, or control whether + to shuffle or stratify the split. See the documentation for more + information: + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Args: + params: Parameters for the data splitter step. + dataset: The dataset to split. + + Returns: + The resulting training and dev subsets. + """ + proccessed_datasets = dataset.remove_columns(["text"]) + proccessed_datasets = proccessed_datasets.rename_column("label", "labels") + proccessed_datasets.set_format("torch") + data_collator = DataCollatorWithPadding(tokenizer=tokenizer) + train_dataloader = DataLoader( + proccessed_datasets["train"], shuffle=True, batch_size=16, collate_fn=data_collator + ) + validation_dataloader = DataLoader( + proccessed_datasets["validation"], batch_size=16, collate_fn=data_collator + ) + return train_dataloader, validation_dataloader \ No newline at end of file diff --git a/template/steps/training/model_trainer.py b/template/steps/training/model_trainer.py index 778adc1..0b26d77 100644 --- a/template/steps/training/model_trainer.py +++ b/template/steps/training/model_trainer.py @@ -32,6 +32,7 @@ def model_trainer( seed: Optional[int] = 42, learning_rate: Optional[float] = 2e-5, load_best_model_at_end: Optional[bool] = True, + evaluation_strategy = "no", eval_batch_size: Optional[int] = 16, weight_decay: Optional[float] = 0.01, ) -> PreTrainedModel: @@ -81,7 +82,10 @@ def model_trainer( per_device_eval_batch_size=eval_batch_size, num_train_epochs=num_epochs, weight_decay=weight_decay, - save_strategy="epoch", + save_strategy='epoch', + evaluation_strategy = "epoch", + #save_steps=200, + save_total_limit=5, load_best_model_at_end=load_best_model_at_end, ) trainer = Trainer( @@ -102,7 +106,10 @@ def model_trainer( per_device_eval_batch_size=eval_batch_size, num_train_epochs=num_epochs, weight_decay=weight_decay, - save_strategy="epoch", + save_strategy='epoch', + evaluation_strategy = "epoch", + #save_steps=200, + save_total_limit=5, load_best_model_at_end=load_best_model_at_end, ) trainer = Trainer( diff --git a/template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} b/template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} new file mode 100644 index 0000000..71d2e67 --- /dev/null +++ b/template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} @@ -0,0 +1,80 @@ +# {% include 'template/license_header' %} + + +import torch +from torch import nn +from transformers import AdamW +from transformers import get_scheduler +from tqdm.auto import tqdm + +from zenml import step +from zenml.enums import StrEnum +from zenml.client import Client + +experiment_tracker = Client().active_stack.experiment_tracker + + +@step +def full_evaluation_step( + evaluation_dataloader: DataLoader, + model: nn.Module, +) -> nn.Module: + """Data splitter step. + + This is an example of a data splitter step that splits the dataset into + training and dev subsets to be used for model training and evaluation. It + takes in a dataset as an step input artifact and returns the training and + dev subsets as two separate step output artifacts. + + Data splitter steps should have a deterministic behavior, i.e. they should + use a fixed random seed and always return the same split when called with + the same input dataset. This is to ensure reproducibility of your pipeline + runs. + + This step is parameterized using the `DataSplitterStepParameters` class, + which allows you to configure the step independently of the step code, + before running it in a pipeline. In this example, the step can be configured + to use a different random seed, change the split ratio, or control whether + to shuffle or stratify the split. See the documentation for more + information: + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Args: + params: Parameters for the data splitter step. + dataset: The dataset to split. + + Returns: + The resulting training and dev subsets. + """ + mlflow.pytorch.autolog() + model = BertForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=3) + optimizer = AdamW(model.parameters(), lr=5e-5) + num_epochs = 5 + num_training_steps = num_epochs * len(train_dataloader) + lr_scheduler = get_scheduler( + "linear", + optimizer=optimizer, + num_warmup_steps=0, + num_training_steps=num_training_steps, + ) + device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + model.to(device) + progress_bar = tqdm(range(num_training_steps)) + model.train() + for epoch in range(num_epochs): + for batch in train_dataloader: + batch = {k: v.to(device) for k, v in batch.items()} + outputs = model(**batch) + loss = outputs.loss + loss.backward() + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + progress_bar.update(1) + + loss, current = loss.item(), batch * len(X) + logger.info(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") + + return model \ No newline at end of file diff --git a/template/steps/training/{% if custom_training %}full_training.py{% endif %} b/template/steps/training/{% if custom_training %}full_training.py{% endif %} new file mode 100644 index 0000000..823cc80 --- /dev/null +++ b/template/steps/training/{% if custom_training %}full_training.py{% endif %} @@ -0,0 +1,79 @@ +# {% include 'template/license_header' %} + + +import torch +from torch import nn +from transformers import AdamW +from transformers import get_scheduler +from tqdm.auto import tqdm + +from zenml import step +from zenml.enums import StrEnum +from zenml.client import Client + +experiment_tracker = Client().active_stack.experiment_tracker + +@step +def full_training_step( + train_dataloader: DataLoader, + hf_pretrained_model: HFPretrainedModel, +) -> nn.Module: + """Data splitter step. + + This is an example of a data splitter step that splits the dataset into + training and dev subsets to be used for model training and evaluation. It + takes in a dataset as an step input artifact and returns the training and + dev subsets as two separate step output artifacts. + + Data splitter steps should have a deterministic behavior, i.e. they should + use a fixed random seed and always return the same split when called with + the same input dataset. This is to ensure reproducibility of your pipeline + runs. + + This step is parameterized using the `DataSplitterStepParameters` class, + which allows you to configure the step independently of the step code, + before running it in a pipeline. In this example, the step can be configured + to use a different random seed, change the split ratio, or control whether + to shuffle or stratify the split. See the documentation for more + information: + + https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + + Args: + params: Parameters for the data splitter step. + dataset: The dataset to split. + + Returns: + The resulting training and dev subsets. + """ + mlflow.pytorch.autolog() + model = BertForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=3) + optimizer = AdamW(model.parameters(), lr=5e-5) + num_epochs = 5 + num_training_steps = num_epochs * len(train_dataloader) + lr_scheduler = get_scheduler( + "linear", + optimizer=optimizer, + num_warmup_steps=0, + num_training_steps=num_training_steps, + ) + device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") + model.to(device) + progress_bar = tqdm(range(num_training_steps)) + model.train() + for epoch in range(num_epochs): + for batch in train_dataloader: + batch = {k: v.to(device) for k, v in batch.items()} + outputs = model(**batch) + loss = outputs.loss + loss.backward() + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + progress_bar.update(1) + + loss, current = loss.item(), batch * len(X) + logger.info(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") + + return model \ No newline at end of file From 6e265e9abe09bedf57f3d2345ce20f0594119552 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:01:05 +0100 Subject: [PATCH 03/18] finilize use case 2 --- template/artifacts/materializer.py | 89 ------- template/artifacts/model_metadata.py | 62 ----- template/config.yaml | 32 +++ template/gradio/Dockerfile | 26 ++ template/{artifacts => gradio}/__init__.py | 0 template/gradio/app.py | 56 +++++ template/pipelines/promoting.py | 64 +++++ template/pipelines/training.py | 66 ++---- template/run.py | 4 +- template/steps/__init__.py | 1 + template/steps/alerts/notify_on.py | 19 +- template/steps/dataset_loader/data_loader.py | 55 ++--- ...ining %}prepare_data_loaders.py{% endif %} | 64 ----- template/steps/inference/__init__.py | 5 - .../inference_get_current_version.py | 34 --- template/steps/inference/inference_predict.py | 43 ---- .../steps/promotion/promote_get_versions.py | 13 +- ...omotion %}promote_get_metric.py{% endif %} | 50 ++++ ...mote_metric_compare_promoter.py{% endif %} | 86 +++++++ ..._promotion %}promote_latest.py{% endif %}} | 11 +- template/steps/registrer/log_register.py | 64 +++++ .../tokenizer_loader/tokenizer_loader.py | 65 ++--- template/steps/tokenzation/tokenization.py | 80 ++++--- template/steps/training/model_trainer.py | 224 ++++++++++-------- ...m_training %}full_evaluation.py{% endif %} | 80 ------- ...tom_training %}full_training.py{% endif %} | 79 ------ template/utils/misc.py | 53 +++-- 27 files changed, 695 insertions(+), 730 deletions(-) delete mode 100644 template/artifacts/materializer.py delete mode 100644 template/artifacts/model_metadata.py create mode 100644 template/config.yaml create mode 100644 template/gradio/Dockerfile rename template/{artifacts => gradio}/__init__.py (100%) create mode 100644 template/gradio/app.py create mode 100644 template/pipelines/promoting.py delete mode 100644 template/steps/dataset_loader/{% if custom_training %}prepare_data_loaders.py{% endif %} delete mode 100644 template/steps/inference/__init__.py delete mode 100644 template/steps/inference/inference_get_current_version.py delete mode 100644 template/steps/inference/inference_predict.py create mode 100644 template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} create mode 100644 template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} rename template/steps/promotion/{promote_latest.py => {% if not metric_compare_promotion %}promote_latest.py{% endif %}} (79%) create mode 100644 template/steps/registrer/log_register.py delete mode 100644 template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} delete mode 100644 template/steps/training/{% if custom_training %}full_training.py{% endif %} diff --git a/template/artifacts/materializer.py b/template/artifacts/materializer.py deleted file mode 100644 index 40c89a3..0000000 --- a/template/artifacts/materializer.py +++ /dev/null @@ -1,89 +0,0 @@ -# {% include 'template/license_header' %} - - -import json -import os -from typing import Type - -from zenml.enums import ArtifactType -from zenml.io import fileio -from zenml.materializers.base_materializer import BaseMaterializer - -from artifacts.model_metadata import ModelMetadata - - -class ModelMetadataMaterializer(BaseMaterializer): - ASSOCIATED_TYPES = (ModelMetadata,) - ASSOCIATED_ARTIFACT_TYPE = ArtifactType.STATISTICS - - def load(self, data_type: Type[ModelMetadata]) -> ModelMetadata: - """Read from artifact store. - - Args: - data_type: What type the artifact data should be loaded as. - - Raises: - ValueError: on deserialization issue - - Returns: - Read artifact. - """ - super().load(data_type) - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - import sklearn.ensemble - import sklearn.linear_model - import sklearn.tree - - modules = [sklearn.ensemble, sklearn.linear_model, sklearn.tree] - - with fileio.open(os.path.join(self.uri, "data.json"), "r") as f: - data_json = json.loads(f.read()) - class_name = data_json["model_class"] - cls = None - for module in modules: - if cls := getattr(module, class_name, None): - break - if cls is None: - raise ValueError( - f"Cannot deserialize `{class_name}` using {self.__class__.__name__}. " - f"Only classes from modules {[m.__name__ for m in modules]} " - "are supported" - ) - data = ModelMetadata(cls) - if "search_grid" in data_json: - data.search_grid = data_json["search_grid"] - if "params" in data_json: - data.params = data_json["params"] - if "metric" in data_json: - data.metric = data_json["metric"] - ### YOUR CODE ENDS HERE ### - - return data - - def save(self, data: ModelMetadata) -> None: - """Write to artifact store. - - Args: - data: The data of the artifact to save. - """ - super().save(data) - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - # Dump the model metadata directly into the artifact store as a JSON file - data_json = dict() - with fileio.open(os.path.join(self.uri, "data.json"), "w") as f: - data_json["model_class"] = data.model_class.__name__ - if data.search_grid: - data_json["search_grid"] = {} - for k, v in data.search_grid.items(): - if type(v) == range: - data_json["search_grid"][k] = list(v) - else: - data_json["search_grid"][k] = v - if data.params: - data_json["params"] = data.params - if data.metric: - data_json["metric"] = data.metric - f.write(json.dumps(data_json)) - ### YOUR CODE ENDS HERE ### diff --git a/template/artifacts/model_metadata.py b/template/artifacts/model_metadata.py deleted file mode 100644 index d3e6ea6..0000000 --- a/template/artifacts/model_metadata.py +++ /dev/null @@ -1,62 +0,0 @@ -# {% include 'template/license_header' %} - - -from typing import Any, Dict - -from sklearn.base import ClassifierMixin - - -class ModelMetadata: - """A custom artifact that stores model metadata. - - A model metadata object gathers together information that is collected - about the model being trained in a training pipeline run. This data type - is used for one of the artifacts returned by the model evaluation step. - - This is an example of a *custom artifact data type*: a type returned by - one of the pipeline steps that isn't natively supported by the ZenML - framework. Custom artifact data types are a common occurrence in ZenML, - usually encountered in one of the following circumstances: - - - you use a third party library that is not covered as a ZenML integration - and you model one or more step artifacts from the data types provided by - this library (e.g. datasets, models, data validation profiles, model - evaluation results/reports etc.) - - you need to use one of your own data types as a step artifact and it is - not one of the basic Python artifact data types supported by the ZenML - framework (e.g. str, int, float, dictionaries, lists, etc.) - - you want to extend one of the artifact data types already natively - supported by ZenML (e.g. pandas.DataFrame or sklearn.ClassifierMixin) - to customize it with your own data and/or behavior. - - In all above cases, the ZenML framework lacks one very important piece of - information: it doesn't "know" how to convert the data into a format that - can be saved in the artifact store (e.g. on a filesystem or persistent - storage service like S3 or GCS). Saving and loading artifacts from the - artifact store is something called "materialization" in ZenML terms and - you need to provide this missing information in the form of a custom - materializer - a class that implements loading/saving artifacts from/to - the artifact store. Take a look at the `materializers` folder to see how a - custom materializer is implemented for this artifact data type. - - More information about custom step artifact data types and ZenML - materializers is available in the docs: - - https://docs.zenml.io/user-guide/advanced-guide/artifact-management/handle-custom-data-types - - """ - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - def __init__( - self, - model_class: ClassifierMixin, - search_grid: Dict[str, Any] = None, - params: Dict[str, Any] = None, - metric: float = None, - ) -> None: - self.model_class = model_class - self.search_grid = search_grid - self.params = params - self.metric = metric - - ### YOUR CODE ENDS HERE ### diff --git a/template/config.yaml b/template/config.yaml new file mode 100644 index 0000000..39fa795 --- /dev/null +++ b/template/config.yaml @@ -0,0 +1,32 @@ +# {% include 'template/license_header' %} + +settings: + docker: + parent_image: 'huggingface/transformers-pytorch-gpu' + required_integrations: +{%- if cloud_of_choice == 'aws' %} + - aws + - skypilot_aws + - s3 +{%- endif %} +{%- if cloud_of_choice == 'gcp' %} + - gcp + - skypilot_gcp +{%- endif %} + - huggingface + - pytorch + - mlflow + - discord + requirements: + - accelerate + - zenml[server] + +extra: + mlflow_model_name: nlp_use_case_model +{%- if target_environment == 'production' %} + target_env: Production +{%- else %} + target_env: Staging +{%- endif %} + notify_on_success: False + notify_on_failure: True \ No newline at end of file diff --git a/template/gradio/Dockerfile b/template/gradio/Dockerfile new file mode 100644 index 0000000..0f09cfb --- /dev/null +++ b/template/gradio/Dockerfile @@ -0,0 +1,26 @@ +# read the doc: https://huggingface.co/docs/hub/spaces-sdks-docker +# you will also find guides on how best to write your Dockerfile + +FROM python:3.9 + +WORKDIR /code + +COPY ./requirements.txt /code/requirements.txt + +RUN pip install --no-cache-dir --upgrade -r /code/requirements.txt + +# Set up a new user named "user" with user ID 1000 +RUN useradd -m -u 1000 user +# Switch to the "user" user +USER user +# Set home to the user's home directory +ENV HOME=/home/user \ + PATH=/home/user/.local/bin:$PATH + +# Set the working directory to the user's home directory +WORKDIR $HOME/app + +# Copy the current directory contents into the container at $HOME/app setting the owner to the user +COPY --chown=user . $HOME/app + +CMD ["python", "app.py", "--server.port=7860", "--server.address=0.0.0.0"] \ No newline at end of file diff --git a/template/artifacts/__init__.py b/template/gradio/__init__.py similarity index 100% rename from template/artifacts/__init__.py rename to template/gradio/__init__.py diff --git a/template/gradio/app.py b/template/gradio/app.py new file mode 100644 index 0000000..8ec5db1 --- /dev/null +++ b/template/gradio/app.py @@ -0,0 +1,56 @@ +import click +import gradio as gr +from transformers import AutoTokenizer, AutoModelForSequenceClassification +import numpy as np + +@click.command() +@click.option('--tokenizer_name', default='roberta-base', help='Name of the tokenizer.') +@click.option('--model_name', default='bert', help='Name of the model.') +@click.option('--labels', default='Negative,Positive', help='Comma-separated list of labels.') +@click.option('--title', default='ZenML NLP Use-Case', help='Title of the Gradio interface.') +@click.option('--description', default='Tweets Analyzer', help='Description of the Gradio interface.') +@click.option('--interpretation', default='default', help='Interpretation mode for the Gradio interface.') +@click.option('--examples', default='bert,This airline sucks -_-', help='Comma-separated list of examples to show in the Gradio interface.') +def sentiment_analysis(tokenizer_name, model_name, labels, title, description, interpretation, examples): + labels = labels.split(',') + examples = [examples.split(',')] + + def preprocess(text): + new_text = [] + for t in text.split(" "): + t = "@user" if t.startswith("@") and len(t) > 1 else t + t = "http" if t.startswith("http") else t + new_text.append(t) + return " ".join(new_text) + + def softmax(x): + e_x = np.exp(x - np.max(x)) + return e_x / e_x.sum(axis=0) + + def analyze_text(text): + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, do_lower_case=True) + model = AutoModelForSequenceClassification.from_pretrained(model_name) + + text = preprocess(text) + encoded_input = tokenizer(text, return_tensors="pt") + output = model(**encoded_input) + scores_ = output[0][0].detach().numpy() + scores_ = softmax(scores_) + + scores = {l: float(s) for (l, s) in zip(labels, scores_)} + return scores + + demo = gr.Interface( + fn=analyze_text, + inputs=[gr.TextArea("Write your text or tweet here", label="Analyze Text")], + outputs=["label"], + title=title, + description=description, + interpretation=interpretation, + examples=examples + ) + + demo.launch(share=True, debug=True) + +if __name__ == '__main__': + sentiment_analysis() \ No newline at end of file diff --git a/template/pipelines/promoting.py b/template/pipelines/promoting.py new file mode 100644 index 0000000..c3bed5c --- /dev/null +++ b/template/pipelines/promoting.py @@ -0,0 +1,64 @@ +# {% include 'template/license_header' %} + +from typing import Optional + +from steps import ( + notify_on_failure, + notify_on_success, +{%- if metric_compare_promotion %} + promote_get_metric, + promote_metric_compare_promoter, +{%- else %} + promote_latest, +{%- endif %} + promote_get_versions, +) +from zenml import pipeline, get_pipeline_context +from zenml.logger import get_logger + + +logger = get_logger(__name__) + +@pipeline( + on_failure=notify_on_failure, +) +def {{product}}_promote_{{dataset}}(): + """ + Model promotion pipeline. + + + + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Link all the steps together by calling them and passing the output + # of one step as the input of the next step. + pipeline_extra = get_pipeline_context().extra + ########## Promotion stage ########## + latest_version, current_version = promote_get_versions() +{%- if metric_compare_promotion %} + latest_metric = promote_get_metric( + metric="accuracy", + version=latest_version, + ) + current_metric = promote_get_metric( + metric="accuracy", + version=current_version, + ) + + promote_metric_compare_promoter( + latest_metric=latest_metric, + current_metric=current_metric, + latest_version=latest_version, + current_version=current_version, + ) + last_step_name = "promote_metric_compare_promoter" +{%- else %} + promote_latest( + latest_version=latest_version, + current_version=current_version, + ) + last_step_name = "promote_latest" +{%- endif %} + + notify_on_success(after=[last_step_name]) + ### YOUR CODE ENDS HERE ### diff --git a/template/pipelines/training.py b/template/pipelines/training.py index 5d01138..b3f8adc 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -1,48 +1,34 @@ # {% include 'template/license_header' %} -from typing import List, Optional +from typing import Optional -from config import DEFAULT_PIPELINE_EXTRAS, PIPELINE_SETTINGS, MetaConfig from steps import ( - data_loader, - model_trainer, notify_on_failure, notify_on_success, - promote_latest, - promote_get_versions, - tokenization_step, + data_loader, tokenizer_loader, + tokenization_step, + model_trainer, + log_register, ) -from zenml import pipeline -from zenml.integrations.mlflow.steps.mlflow_deployer import ( - mlflow_model_registry_deployer_step, -) -from zenml.integrations.mlflow.steps.mlflow_registry import mlflow_register_model_step +from zenml import pipeline, get_pipeline_context from zenml.logger import get_logger -from zenml.steps.external_artifact import ExternalArtifact logger = get_logger(__name__) -@pipeline( - settings=PIPELINE_SETTINGS, - on_failure=notify_on_failure, - extra=DEFAULT_PIPELINE_EXTRAS, -) -def {{product_name}}_training( - #hf_dataset: HFSentimentAnalysisDataset, - #hf_tokenizer: HFPretrainedTokenizer, - #hf_pretrained_model: HFPretrainedModel, +@pipeline(on_failure=notify_on_failure) +def {{product_name}}_training_{{dataset}}( lower_case: Optional[bool] = True, padding: Optional[str] = "max_length", max_seq_length: Optional[int] = 128, text_column: Optional[str] = "text", label_column: Optional[str] = "label", - train_batch_size: Optional[int] = 16, - eval_batch_size: Optional[int] = 16, - num_epochs: Optional[int] = 3, + train_batch_size: Optional[int] = 8, + eval_batch_size: Optional[int] = 8, + num_epochs: Optional[int] = 5, learning_rate: Optional[float] = 2e-5, weight_decay: Optional[float] = 0.01, ): @@ -68,12 +54,12 @@ def {{product_name}}_training( ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### # Link all the steps together by calling them and passing the output # of one step as the input of the next step. + pipeline_extra = get_pipeline_context().extra ########## Tokenization stage ########## dataset = data_loader( - hf_dataset=MetaConfig.dataset, + shuffle=True, ) tokenizer = tokenizer_loader( - hf_tokenizer=MetaConfig.tokenizer, lower_case=lower_case ) tokenized_data = tokenization_step( @@ -82,14 +68,12 @@ def {{product_name}}_training( padding=padding, max_seq_length=max_seq_length, text_column=text_column, - label_column=label_column + label_column=label_column, ) - ########## Training stage ########## - model = model_trainer( + model, tokenizer = model_trainer( tokenized_dataset=tokenized_data, - hf_pretrained_model=MetaConfig.model, tokenizer=tokenizer, train_batch_size=train_batch_size, eval_batch_size=eval_batch_size, @@ -98,20 +82,12 @@ def {{product_name}}_training( weight_decay=weight_decay, ) - mlflow_register_model_step( - model, - name=MetaConfig.mlflow_model_name, - ) - - ########## Promotion stage ########## - latest_version, current_version = promote_get_versions( - after=["mlflow_register_model_step"], - ) - promote_latest( - latest_version=latest_version, - current_version=current_version, + ########## Log and Register stage ########## + log_register( + model=model, + tokenizer=tokenizer, + name="{{product_name}}_training_{{dataset}}", ) - last_step_name = "promote_latest" - notify_on_success(after=[last_step_name]) + notify_on_success(after=[log_register]) ### YOUR CODE ENDS HERE ### diff --git a/template/run.py b/template/run.py index 9ae1419..98bdbd9 100644 --- a/template/run.py +++ b/template/run.py @@ -4,9 +4,7 @@ from zenml.steps.external_artifact import ExternalArtifact from zenml.logger import get_logger from pipelines import {{product_name}}_training -from config import MetaConfig import click -from typing import Optional from datetime import datetime as dt logger = get_logger(__name__) @@ -125,7 +123,7 @@ def main( pipeline_args[ "run_name" - ] = f"{MetaConfig.pipeline_name_training}_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + ] = f"{{product_name}}_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" {{product_name}}_training.with_options(**pipeline_args)(**run_args_train) logger.info("Training pipeline finished successfully!") diff --git a/template/steps/__init__.py b/template/steps/__init__.py index 3f178a5..c19d1a2 100644 --- a/template/steps/__init__.py +++ b/template/steps/__init__.py @@ -17,3 +17,4 @@ promote_get_versions, ) from .training import model_trainer +from .registrer import log_register diff --git a/template/steps/alerts/notify_on.py b/template/steps/alerts/notify_on.py index ae183a9..33ad394 100644 --- a/template/steps/alerts/notify_on.py +++ b/template/steps/alerts/notify_on.py @@ -1,4 +1,19 @@ -# {% include 'template/license_header' %} +# Apache Software License 2.0 +# +# Copyright (c) ZenML GmbH 2023. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from zenml import get_step_context, step @@ -39,4 +54,4 @@ def notify_on_success() -> None: """Notifies user on pipeline success.""" step_context = get_step_context() if alerter and step_context.pipeline_run.config.extra["notify_on_success"]: - alerter.post(message=build_message(status="succeeded")) + alerter.post(message=build_message(status="succeeded")) \ No newline at end of file diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index 1d2a7cd..1786ae5 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -1,62 +1,57 @@ # {% include 'template/license_header' %} - from typing_extensions import Annotated - from datasets import load_dataset, DatasetDict - from zenml import step from zenml.logger import get_logger -from config import HFSentimentAnalysisDataset - logger = get_logger(__name__) - @step def data_loader( - hf_dataset: HFSentimentAnalysisDataset, + shuffle: bool = True, ) -> Annotated[DatasetDict, "dataset"]: - """Data loader step. + """ + Data loader step. - This is an example of a data loader step that is usually the first step - in your pipeline. It reads data from an external source like a file, + This step reads data from an external source like a file, database or 3rd party library, then formats it and returns it as a step output artifact. - This step is parameterized using the `DataLoaderStepParameters` class, which + This step is parameterized using the `HFSentimentAnalysisDataset` class, which allows you to configure the step independently of the step code, before running it in a pipeline. In this example, the step can be configured to - load different built-in scikit-learn datasets. See the documentation for - more information: - - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + load different built-in scikit-learn datasets. Data loader steps should have caching disabled if they are not deterministic (i.e. if they data they load from the external source can be different when they are subsequently called, even if the step code and parameter values don't change). - Args: - params: Parameters for the data loader step. - Returns: The loaded dataset artifact. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - # Load the dataset indicated in the step parameters and format it as a - # pandas DataFrame - logger.info(f"Loaded dataset {hf_dataset.value}") - if ( - hf_dataset == HFSentimentAnalysisDataset.financial_news - or hf_dataset != HFSentimentAnalysisDataset.imbd_reviews - and hf_dataset == HFSentimentAnalysisDataset.airline_reviews - ): - dataset = load_dataset(hf_dataset.value) - elif hf_dataset == HFSentimentAnalysisDataset.imbd_reviews: - dataset = load_dataset(hf_dataset.value, split='train') + logger.info(f"Loaded dataset {{dataset}}") + + # Load dataset based on the dataset value + {%- if dataset == 'financial_news' %} + dataset = load_dataset({{dataset}}) + {%- endif %} + {%- if dataset == 'imbd_reviews' %} + dataset = load_dataset({{dataset}})["train"] + dataset = dataset.train_test_split(test_size=0.25, shuffle=shuffle) + {%- endif %} + {%- if dataset == 'airline_reviews' %} + dataset = load_dataset({{dataset}}) + dataset = dataset.rename_column("airline_sentiment", "label") + dataset = dataset.remove_columns(["airline_sentiment_confidence","negativereason_confidence"]) + {%- endif %} + + # Log the dataset and sample examples logger.info(dataset) + logger.info("Sample Example :", dataset["train"][0]) logger.info("Sample Example :", dataset["train"][1]) ### YOUR CODE ENDS HERE ### - + return dataset \ No newline at end of file diff --git a/template/steps/dataset_loader/{% if custom_training %}prepare_data_loaders.py{% endif %} b/template/steps/dataset_loader/{% if custom_training %}prepare_data_loaders.py{% endif %} deleted file mode 100644 index 7c6f5ac..0000000 --- a/template/steps/dataset_loader/{% if custom_training %}prepare_data_loaders.py{% endif %} +++ /dev/null @@ -1,64 +0,0 @@ -# {% include 'template/license_header' %} - - -from typing_extensions import Annotated - -from torch.utils.data import DataLoader - -from zenml import step -from zenml.logger import get_logger - -from config import HFSentimentAnalysisDataset - -logger = get_logger(__name__) - - -@step -def prepare_dataloaders_step( - tokenizer: PreTrainedTokenizerBase, - dataset: DatasetDict, -) -> ( - Tuple[ - Annotated[DataLoader, "train_dataloader"], - Annotated[DataLoader, "validation_dataloader"], - ] -): - """Data splitter step. - - This is an example of a data splitter step that splits the dataset into - training and dev subsets to be used for model training and evaluation. It - takes in a dataset as an step input artifact and returns the training and - dev subsets as two separate step output artifacts. - - Data splitter steps should have a deterministic behavior, i.e. they should - use a fixed random seed and always return the same split when called with - the same input dataset. This is to ensure reproducibility of your pipeline - runs. - - This step is parameterized using the `DataSplitterStepParameters` class, - which allows you to configure the step independently of the step code, - before running it in a pipeline. In this example, the step can be configured - to use a different random seed, change the split ratio, or control whether - to shuffle or stratify the split. See the documentation for more - information: - - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions - - Args: - params: Parameters for the data splitter step. - dataset: The dataset to split. - - Returns: - The resulting training and dev subsets. - """ - proccessed_datasets = dataset.remove_columns(["text"]) - proccessed_datasets = proccessed_datasets.rename_column("label", "labels") - proccessed_datasets.set_format("torch") - data_collator = DataCollatorWithPadding(tokenizer=tokenizer) - train_dataloader = DataLoader( - proccessed_datasets["train"], shuffle=True, batch_size=16, collate_fn=data_collator - ) - validation_dataloader = DataLoader( - proccessed_datasets["validation"], batch_size=16, collate_fn=data_collator - ) - return train_dataloader, validation_dataloader \ No newline at end of file diff --git a/template/steps/inference/__init__.py b/template/steps/inference/__init__.py deleted file mode 100644 index fc63455..0000000 --- a/template/steps/inference/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# {% include 'template/license_header' %} - - -from .inference_get_current_version import inference_get_current_version -from .inference_predict import inference_predict diff --git a/template/steps/inference/inference_get_current_version.py b/template/steps/inference/inference_get_current_version.py deleted file mode 100644 index 0454d87..0000000 --- a/template/steps/inference/inference_get_current_version.py +++ /dev/null @@ -1,34 +0,0 @@ -# {% include 'template/license_header' %} - - -from typing_extensions import Annotated - -from config import MetaConfig -from zenml import step -from zenml.client import Client -from zenml.logger import get_logger - -logger = get_logger(__name__) - -model_registry = Client().active_stack.model_registry - - -@step -def inference_get_current_version() -> Annotated[str, "model_version"]: - """Get currently tagged model version for deployment. - - Returns: - The model version of currently tagged model in Registry. - """ - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - - current_version = model_registry.list_model_versions( - name=MetaConfig.mlflow_model_name, - stage=MetaConfig.target_env, - )[0].version - logger.info( - f"Current model version in `{MetaConfig.target_env.value}` is `{current_version}`" - ) - - return current_version diff --git a/template/steps/inference/inference_predict.py b/template/steps/inference/inference_predict.py deleted file mode 100644 index 3406583..0000000 --- a/template/steps/inference/inference_predict.py +++ /dev/null @@ -1,43 +0,0 @@ -# {% include 'template/license_header' %} - - -from typing_extensions import Annotated - -import pandas as pd -from zenml import step -from zenml.integrations.mlflow.model_deployers.mlflow_model_deployer import ( - MLFlowDeploymentService, -) - - -@step -def inference_predict( - deployment_service: MLFlowDeploymentService, - dataset_inf: pd.DataFrame, -) -> Annotated[pd.Series, "predictions"]: - """Predictions step. - - This is an example of a predictions step that takes the data in and returns - predicted values. - - This step is parameterized, which allows you to configure the step - independently of the step code, before running it in a pipeline. - In this example, the step can be configured to use different input data - and model version in registry. See the documentation for more information: - - https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines - - Args: - deployment_service: Deployed model service. - dataset_inf: The inference dataset. - - Returns: - The processed dataframe: dataset_inf. - """ - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - predictions = deployment_service.predict(request=dataset_inf) - predictions = pd.Series(predictions, name="predicted") - deployment_service.deprovision(force=True) - ### YOUR CODE ENDS HERE ### - - return predictions diff --git a/template/steps/promotion/promote_get_versions.py b/template/steps/promotion/promote_get_versions.py index 4366a37..c7042cb 100644 --- a/template/steps/promotion/promote_get_versions.py +++ b/template/steps/promotion/promote_get_versions.py @@ -4,10 +4,10 @@ from typing import Tuple from typing_extensions import Annotated -from config import MetaConfig -from zenml import step +from zenml import step, get_step_context from zenml.client import Client from zenml.logger import get_logger +from zenml.model_registries.base_model_registry import ModelVersionStage logger = get_logger(__name__) @@ -30,16 +30,17 @@ def promote_get_versions() -> ( """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + pipeline_extra = get_step_context().pipeline_run.config.extra none_versions = model_registry.list_model_versions( - name=MetaConfig.mlflow_model_name, + name=pipeline_extra["mlflow_model_name"], stage=None, ) latest_versions = none_versions[0].version logger.info(f"Latest model version is {latest_versions}") target_versions = model_registry.list_model_versions( - name=MetaConfig.mlflow_model_name, - stage=MetaConfig.target_env, + name=pipeline_extra["mlflow_model_name"], + stage=ModelVersionStage(pipeline_extra["target_env"]), ) current_version = latest_versions if target_versions: @@ -47,4 +48,6 @@ def promote_get_versions() -> ( logger.info(f"Currently promoted model version is {current_version}") else: logger.info("No currently promoted model version found.") + ### YOUR CODE ENDS HERE ### + return current_version, current_version diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} new file mode 100644 index 0000000..9e0b13c --- /dev/null +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} @@ -0,0 +1,50 @@ +# {% include 'template/license_header' %} + + +from typing_extensions import Annotated + +import pandas as pd +from sklearn.metrics import accuracy_score +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger +from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker + +logger = get_logger(__name__) + +model_registry = Client().active_stack.model_registry + +@step +def promote_get_metric( + metric: str, + version: str, +) -> Annotated[float, "metric"]: + """Get metric for comparison for one model deployment. + + This is an example of a metric calculation step. It get a model deployment + service and computes metric on recent test dataset. + + This step is parameterized, which allows you to configure the step + independently of the step code, before running it in a pipeline. + In this example, the step can be configured to use different input data. + See the documentation for more information: + + https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines + + Args: + dataset_tst: The test dataset. + deployment_service: Model version deployment. + + Returns: + Metric value for a given deployment on test set. + + """ + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + model_version = model_registry.get_model_version(version) + mlflow_run = mlflow.get_run(run_id=model_version.metadata["mlflow_run_id"]) + logger.info("Getting metric from MLFlow run %s", mlflow_run.info.run_id) + + metric = mlflow_run.data.metrics[metric] + ### YOUR CODE ENDS HERE ### + return metric diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} new file mode 100644 index 0000000..d055885 --- /dev/null +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} @@ -0,0 +1,86 @@ +# {% include 'template/license_header' %} + + +from zenml import get_step_context, step +from zenml.client import Client +from zenml.logger import get_logger +from zenml.model_registries.base_model_registry import ModelVersionStage + +logger = get_logger(__name__) + +model_registry = Client().active_stack.model_registry + + +@step +def promote_metric_compare_promoter( + latest_metric: float, + current_metric: float, + latest_version: str, + current_version: str, +): + """Try to promote trained model. + + This is an example of a model promotion step. It gets precomputed + metrics for 2 model version: latest and currently promoted to target environment + (Production, Staging, etc) and compare than in order to define + if newly trained model is performing better or not. If new model + version is better by metric - it will get relevant + tag, otherwise previously promoted model version will remain. + + If the latest version is the only one - it will get promoted automatically. + + This step is parameterized, which allows you to configure the step + independently of the step code, before running it in a pipeline. + In this example, the step can be configured to use different input data. + See the documentation for more information: + + https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines + + Args: + latest_metric: Recently trained model metric results. + current_metric: Previously promoted model metric results. + latest_version: Recently trained model version. + current_version:Previously promoted model version. + + """ + + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + pipeline_extra = get_step_context().pipeline_run.config.extra + should_promote = True + + if latest_version == current_version: + logger.info("No current model version found - promoting latest") + else: + logger.info( + f"Latest model metric={latest_metric:.6f}\n" + f"Current model metric={current_metric:.6f}" + ) + if latest_metric > current_metric: + logger.info( + "Latest model versions outperformed current versions - promoting latest" + ) + else: + logger.info( + "Current model versions outperformed latest versions - keeping current" + ) + should_promote = False + + promoted_version = current_version + if should_promote: + if latest_version != current_version: + model_registry.update_model_version( + name=pipeline_extra["mlflow_model_name"], + version=current_version, + stage=ModelVersionStage.ARCHIVED, + ) + model_registry.update_model_version( + name=pipeline_extra["mlflow_model_name"], + version=latest_version, + stage=ModelVersionStage(pipeline_extra["target_env"]), + ) + promoted_version = latest_version + + logger.info( + f"Current model version in `{pipeline_extra['target_env']}` is `{promoted_version}`" + ) + ### YOUR CODE ENDS HERE ### diff --git a/template/steps/promotion/promote_latest.py b/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} similarity index 79% rename from template/steps/promotion/promote_latest.py rename to template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} index 6b940fe..85aab9f 100644 --- a/template/steps/promotion/promote_latest.py +++ b/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} @@ -6,8 +6,6 @@ from zenml.logger import get_logger from zenml.model_registries.base_model_registry import ModelVersionStage -from config import MetaConfig - logger = get_logger(__name__) model_registry = Client().active_stack.model_registry @@ -28,22 +26,23 @@ def promote_latest(latest_version:str, current_version:str): ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### logger.info(f"Promoting latest model version `{latest_version}`") + pipeline_extra = get_step_context().pipeline_run.config.extra if latest_version != current_version: model_registry.update_model_version( - name=MetaConfig.mlflow_model_name, + name=pipeline_extra["mlflow_model_name"], version=current_version, stage=ModelVersionStage.ARCHIVED, metadata={}, ) model_registry.update_model_version( - name=MetaConfig.mlflow_model_name, + name=pipeline_extra["mlflow_model_name"], version=latest_version, - stage=MetaConfig.target_env, + stage=pipeline_extra["target_env"], metadata={}, ) promoted_version = latest_version logger.info( - f"Current model version in `{MetaConfig.target_env.value}` is `{promoted_version}`" + f"Current model version in `{pipeline_extra['target_env']}` is `{promoted_version}`" ) ### YOUR CODE ENDS HERE ### diff --git a/template/steps/registrer/log_register.py b/template/steps/registrer/log_register.py new file mode 100644 index 0000000..41a195c --- /dev/null +++ b/template/steps/registrer/log_register.py @@ -0,0 +1,64 @@ +# {% include 'template/license_header' %} + +from typing import Optional +import mlflow +from transformers import ( + PreTrainedModel, + PreTrainedTokenizerBase, +) + +from zenml import step +from zenml.client import Client +from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker +from zenml.logger import get_logger + +# Initialize logger +logger = get_logger(__name__) + +# Get experiment tracker +experiment_tracker = Client().active_stack.experiment_tracker + +# Check if experiment tracker is set and is of type MLFlowExperimentTracker +if not experiment_tracker or not isinstance( + experiment_tracker, MLFlowExperimentTracker +): + raise RuntimeError( + "Your active stack needs to contain a MLFlow experiment tracker for " + "this example to work." + ) + +@step(experiment_tracker=experiment_tracker.name) +def model_register( + model: PreTrainedModel, + tokenizer: PreTrainedTokenizerBase, + mlflow_model_name: Optional[str] = "model", +): + """ + Register model to MLFlow. + + This step takes in a model and tokenizer artifact previously loaded and pre-processed by + other steps in your pipeline, then registers the model to MLFlow for deployment. + + Model training steps should have caching disabled if they are not deterministic + (i.e. if the model training involve some random processes like initializing + weights or shuffling data that are not controlled by setting a fixed random seed). + + Args: + model: The model. + tokenizer: The tokenizer. + + Returns: + The trained model and tokenizer. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Log the model + components = { + "model": model, + "tokenizer": tokenizer, + } + mlflow.transformers.log_model( + transformers_model=components, + artifact_path=mlflow_model_name, + registered_model_name=mlflow_model_name, + ) + ### YOUR CODE ENDS HERE ### \ No newline at end of file diff --git a/template/steps/tokenizer_loader/tokenizer_loader.py b/template/steps/tokenizer_loader/tokenizer_loader.py index 0265690..71b1aba 100644 --- a/template/steps/tokenizer_loader/tokenizer_loader.py +++ b/template/steps/tokenizer_loader/tokenizer_loader.py @@ -1,53 +1,56 @@ # {% include 'template/license_header' %} - +from transformers import PreTrainedTokenizerBase, AutoTokenizer from typing_extensions import Annotated -from transformers import BertTokenizer, GPT2Tokenizer, PreTrainedTokenizerBase - -from zenml.enums import StrEnum from zenml import step from zenml.logger import get_logger logger = get_logger(__name__) -class HFPretrainedTokenizer(StrEnum): - """HuggingFace Sentiment Analysis datasets.""" - bert = "bert-base-uncased" - gpt2 = "gpt2" - @step def tokenizer_loader( - hf_tokenizer: HFPretrainedTokenizer, lower_case: bool, -) -> Annotated[PreTrainedTokenizerBase, "tokenzer"]: - """Tokenizer loader step. +) -> Annotated[PreTrainedTokenizerBase, "tokenizer"]: + """Tokenizer selection step. + + This step is responsible for selecting and initializing the tokenizer based + on the model type. The tokenizer is a crucial component in Natural Language + Processing tasks as it is responsible for converting the input text into a + format that the model can understand. - This is an example of a data processor step that prepares the data so that - it is suitable for model training. It takes in a dataset as an input step - artifact and performs any necessary preprocessing steps like cleaning, - feature engineering, feature selection, etc. It then returns the processed - dataset as a step output artifact. + This step is parameterized, which allows you to configure the step independently + of the step code, before running it in a pipeline. In this example, the step can + be configured to use different types of tokenizers corresponding to different + models such as 'bert', 'roberta', or 'distilbert'. - This step is parameterized using the `DataProcessorStepParameters` class, - which allows you to configure the step independently of the step code, - before running it in a pipeline. In this example, the step can be configured - to perform or skip different preprocessing steps (e.g. dropping rows with - missing values, dropping columns, normalizing the data, etc.). See the - documentation for more information: + For more information on how to configure steps in a pipeline, refer to the + following documentation: - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines Args: - params: Parameters for the data processor step. + lower_case: A boolean value indicating whether to convert the input text to + lower case during tokenization. Returns: - The processed dataset artifact. + The initialized tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - if hf_tokenizer == HFPretrainedTokenizer.bert: - tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=lower_case) - elif hf_tokenizer == HFPretrainedTokenizer.gpt2: - tokenizer = GPT2Tokenizer.from_pretrained("gpt2", bos_token='<|startoftext|>', eos_token='<|endoftext|>', pad_token='<|pad|>') + {%- if model == 'bert' %} + tokenizer = AutoTokenizer.from_pretrained( + "bert-base-uncased", do_lower_case=lower_case + ) + {%- endif %} + {%- if model == 'roberta' %} + tokenizer = AutoTokenizer.from_pretrained( + "roberta-base", do_lower_case=lower_case + ) + {%- endif %} + {%- if model == 'distilbert' %} + tokenizer = AutoTokenizer.from_pretrained( + "distilbert-base-cased", do_lower_case=lower_case + ) + {%- endif %} ### YOUR CODE ENDS HERE ### - return tokenizer \ No newline at end of file + return tokenizer diff --git a/template/steps/tokenzation/tokenization.py b/template/steps/tokenzation/tokenization.py index 629d7db..11a47d4 100644 --- a/template/steps/tokenzation/tokenization.py +++ b/template/steps/tokenzation/tokenization.py @@ -1,38 +1,33 @@ # {% include 'template/license_header' %} - from typing_extensions import Annotated from transformers import PreTrainedTokenizerBase - from datasets import DatasetDict from zenml import step from zenml.logger import get_logger - from utils.misc import find_max_length logger = get_logger(__name__) - @step def tokenization_step( padding: str, - max_seq_length: int, - text_column: str, - label_column: str, + max_seq_length: int = 512, + text_column: str = "text", + label_column: str = "label", tokenizer: PreTrainedTokenizerBase, dataset: DatasetDict, ) -> Annotated[DatasetDict, "tokenized_data"]: - """Data splitter step. + """ + Tokenization step. - This is an example of a data splitter step that splits the dataset into - training and dev subsets to be used for model training and evaluation. It - takes in a dataset as an step input artifact and returns the training and - dev subsets as two separate step output artifacts. + This step tokenizes the input dataset using a pre-trained tokenizer. It + takes in a dataset as an step input artifact and returns the tokenized + dataset as an output artifact. - Data splitter steps should have a deterministic behavior, i.e. they should - use a fixed random seed and always return the same split when called with - the same input dataset. This is to ensure reproducibility of your pipeline - runs. + The tokenization process includes padding, truncation, and addition of + labels. The maximum sequence length for tokenization is determined based + on the dataset. This step is parameterized using the `DataSplitterStepParameters` class, which allows you to configure the step independently of the step code, @@ -44,28 +39,53 @@ def tokenization_step( https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions Args: - params: Parameters for the data splitter step. - dataset: The dataset to split. + padding: Padding method for tokenization. + max_seq_length: Maximum sequence length for tokenization. + text_column: Column name for text data in the dataset. + label_column: Column name for label data in the dataset. + tokenizer: Pre-trained tokenizer for tokenization. + dataset: The dataset to tokenize. Returns: - The resulting training and dev subsets. + The tokenized dataset. """ - train_max_length = find_max_length(dataset["train"]["text"]) - val_max_length = find_max_length(dataset["validation"]["text"]) - max_length = train_max_length if train_max_length >= val_max_length else val_max_length - logger.info(f"max length for the given dataset is:{max_length}") ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + train_max_length = find_max_length(dataset["train"][text_column]) + + # Depending on the dataset, find the maximum length of text in the validation or test dataset +{%- if dataset == 'imdb' %} + val_or_test_max_length = find_max_length(dataset["test"][text_column]) +{%- else %} + val_or_test_max_length = find_max_length(dataset["validation"][text_column]) + max_length = train_max_length if train_max_length >= val_or_test_max_length else val_or_test_max_length +{%- endif %} + logger.info(f"max length for the given dataset is:{max_length}") + + # Determine the maximum length for tokenization + max_length = train_max_length if train_max_length >= val_or_test_max_length else val_or_test_max_length + logger.info(f"max length for the given dataset is:{max_length}") + def preprocess_function(examples): + # Tokenize the examples with padding, truncation, and a specified maximum length result = tokenizer( examples[text_column], - #padding="max_length", + padding="max_length", truncation=True, - max_length=max_length or max_seq_length + max_length=max_length or max_seq_length, ) + # Add labels to the tokenized examples result["label"] = examples[label_column] return result - tokenized_datasets = dataset.map(preprocess_function, batched=True) - #tokenized_datasets = tokenized_datasets.remove_columns(["text"]) - #tokenized_datasets = tokenized_datasets.rename_column("label", "labels") - #tokenized_datasets.set_format("torch") - return tokenized_datasets \ No newline at end of file + + # Apply the preprocessing function to the dataset + tokenized_datasets = dataset.map(preprocess_function, batched=True,) + logger.info(tokenized_datasets) + + # Remove the original text column and rename the label column + tokenized_datasets = tokenized_datasets.remove_columns([text_column]) + tokenized_datasets = tokenized_datasets.rename_column(label_column, "labels") + + # Set the format of the tokenized dataset + tokenized_datasets.set_format("torch") + ### YOUR CODE ENDS HERE ### + return tokenized_datasets diff --git a/template/steps/training/model_trainer.py b/template/steps/training/model_trainer.py index 0b26d77..f7b550e 100644 --- a/template/steps/training/model_trainer.py +++ b/template/steps/training/model_trainer.py @@ -1,130 +1,150 @@ -from typing import Tuple, Optional +# {% include 'template/license_header' %} -from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score -from transformers import TrainingArguments, Trainer, PreTrainedModel, DataCollatorWithPadding -from transformers import BertForSequenceClassification, GPT2ForSequenceClassification -from datasets import DatasetDict -from transformers import PreTrainedTokenizerBase -import mlflow +from typing import Optional, Tuple +from typing_extensions import Annotated +import mlflow +from datasets import DatasetDict +from transformers import ( + DataCollatorWithPadding, + PreTrainedModel, + PreTrainedTokenizerBase, + Trainer, + TrainingArguments, + AutoModelForSequenceClassification, +) from zenml import step -from zenml.enums import StrEnum from zenml.client import Client +from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker +from zenml.logger import get_logger -from utils.misc import compute_metrics +from template.utils.misc import compute_metrics -experiment_tracker = Client().active_stack.experiment_tracker +# Initialize logger +logger = get_logger(__name__) -class HFPretrainedModel(StrEnum): - """HuggingFace Sentiment Analysis Model.""" - bert = "bert-base-uncased" - gpt2 = "gpt2" +# Get experiment tracker +experiment_tracker = Client().active_stack.experiment_tracker +# Check if experiment tracker is set and is of type MLFlowExperimentTracker +if not experiment_tracker or not isinstance( + experiment_tracker, MLFlowExperimentTracker +): + raise RuntimeError( + "Your active stack needs to contain a MLFlow experiment tracker for " + "this example to work." + ) @step(experiment_tracker=experiment_tracker.name) def model_trainer( - hf_pretrained_model: HFPretrainedModel, tokenized_dataset: DatasetDict, tokenizer: PreTrainedTokenizerBase, - num_labels: Optional[int] = 3, + num_labels: Optional[int] = None, train_batch_size: Optional[int] = 16, num_epochs: Optional[int] = 3, - seed: Optional[int] = 42, learning_rate: Optional[float] = 2e-5, load_best_model_at_end: Optional[bool] = True, - evaluation_strategy = "no", eval_batch_size: Optional[int] = 16, weight_decay: Optional[float] = 0.01, -) -> PreTrainedModel: - """Configure and train a model on the training dataset. - - This is an example of a model training step that takes in a dataset artifact - previously loaded and pre-processed by other steps in your pipeline, then - configures and trains a model on it. The model is then returned as a step - output artifact. - - Model training steps should have caching disabled if they are not - deterministic (i.e. if the model training involve some random processes - like initializing weights or shuffling data that are not controlled by - setting a fixed random seed). This example step ensures the outcome is - deterministic by initializing the model with a fixed random seed. - - This step is parameterized using the `ModelTrainerStepParameters` class, - which allows you to configure the step independently of the step code, -{%- if configurable_model %} - before running it in a pipeline. In this example, the step can be configured - to use a different model, change the random seed, or pass different - hyperparameters to the model constructor. See the documentation for more - information: -{%- else %} - before running it in a pipeline. In this example, the step can be configured - to change the random seed, or pass different hyperparameters to the model - constructor. See the documentation for more information: -{%- endif %} - - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + mlflow_model_name: Optional[str] = "model", +) -> Tuple[Annotated[PreTrainedModel, "model"], Annotated[PreTrainedTokenizerBase, "tokenizer"]]: + """ + Configure and train a model on the training dataset. + + This step takes in a dataset artifact previously loaded and pre-processed by + other steps in your pipeline, then configures and trains a model on it. The + model is then returned as a step output artifact. + + Model training steps should have caching disabled if they are not deterministic + (i.e. if the model training involve some random processes like initializing + weights or shuffling data that are not controlled by setting a fixed random seed). + Args: - params: The parameters for the model trainer step. - train_set: The training data set artifact. + hf_pretrained_model: The pre-trained model. + tokenized_dataset: The tokenized dataset. + tokenizer: The tokenizer. + num_labels: The number of labels. + train_batch_size: The training batch size. + num_epochs: The number of epochs. + learning_rate: The learning rate. + load_best_model_at_end: Whether to load the best model at the end. + eval_batch_size: The evaluation batch size. + weight_decay: The weight decay. Returns: - The trained model artifact. + The trained model and tokenizer. """ - mlflow.transformers.autolog() - data_collator = DataCollatorWithPadding(tokenizer=tokenizer) - if hf_pretrained_model == HFPretrainedModel.bert: - model = BertForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=num_labels) - training_args = TrainingArguments( - output_dir="zenml_artifact", - learning_rate=learning_rate, - per_device_train_batch_size=train_batch_size, - per_device_eval_batch_size=eval_batch_size, - num_train_epochs=num_epochs, - weight_decay=weight_decay, - save_strategy='epoch', - evaluation_strategy = "epoch", - #save_steps=200, - save_total_limit=5, - load_best_model_at_end=load_best_model_at_end, - ) - trainer = Trainer( - model=model, - args=training_args, - train_dataset=tokenized_dataset["train"], - eval_dataset=tokenized_dataset["validation"], - compute_metrics=compute_metrics, - data_collator=data_collator, - ) - elif hf_pretrained_model == HFPretrainedModel.gpt2: - model = GPT2ForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=3) - model.resize_token_embeddings(len(tokenizer)) - training_args = TrainingArguments( - output_dir="zenml_artifact", - learning_rate=learning_rate, - per_device_train_batch_size=train_batch_size, - per_device_eval_batch_size=eval_batch_size, - num_train_epochs=num_epochs, - weight_decay=weight_decay, - save_strategy='epoch', - evaluation_strategy = "epoch", - #save_steps=200, - save_total_limit=5, - load_best_model_at_end=load_best_model_at_end, - ) - trainer = Trainer( - model=model, - args=training_args, - train_dataset=tokenized_dataset["train"], - eval_dataset=tokenized_dataset["validation"], - compute_metrics=compute_metrics, - data_collator=data_collator, - ) ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - # Initialize the model with the hyperparameters indicated in the step - # parameters and train it on the training set. + # Select the appropriate datasets based on the dataset type + {%- if dataset == 'imdb' %} + train_dataset = tokenized_dataset['train'] + eval_dataset = tokenized_dataset['test'] + {%- else %} + train_dataset = tokenized_dataset['train'] + eval_dataset = tokenized_dataset['validation'] + {%- endif %} - ### YOUR CODE ENDS HERE ### + # Initialize data collator + data_collator = DataCollatorWithPadding(tokenizer=tokenizer) + + # Set the number of labels + num_labels = num_labels or len(train_dataset.unique("labels")) + + # Set the logging steps + logging_steps = len(train_dataset) // train_batch_size + + # Initialize training arguments + training_args = TrainingArguments( + output_dir="zenml_artifact", + learning_rate=learning_rate, + per_device_train_batch_size=train_batch_size, + per_device_eval_batch_size=eval_batch_size, + num_train_epochs=num_epochs, + weight_decay=weight_decay, + evaluation_strategy='steps', + save_strategy='steps', + logging_steps=logging_steps, + save_total_limit=5, + report_to="mlflow", + load_best_model_at_end=load_best_model_at_end, + ) + logger.info(f"Training arguments: {training_args}") + + # Load the model + model = AutoModelForSequenceClassification.from_pretrained( + {{model}}, num_labels=num_labels + ) + + # Enable autologging + mlflow.transformers.autolog() + + # Initialize the trainer + trainer = Trainer( + model=model, + args=training_args, + train_dataset=train_dataset, + eval_dataset=eval_dataset, + compute_metrics=compute_metrics, + data_collator=data_collator, + ) + + # Train and evaluate the model trainer.train() trainer.evaluate() - return model \ No newline at end of file + + #{%- if log_at_trainer %} + # Log the model + #components = { + # "model": model, + # "tokenizer": tokenizer, + #} + #mlflow.transformers.log_model( + # transformers_model=components, + # artifact_path=mlflow_model_name, + # registered_model_name=mlflow_model_name, + #) + #{%- endif %} + ### YOUR CODE ENDS HERE ### + + return model, tokenizer \ No newline at end of file diff --git a/template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} b/template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} deleted file mode 100644 index 71d2e67..0000000 --- a/template/steps/training/{% if custom_training %}full_evaluation.py{% endif %} +++ /dev/null @@ -1,80 +0,0 @@ -# {% include 'template/license_header' %} - - -import torch -from torch import nn -from transformers import AdamW -from transformers import get_scheduler -from tqdm.auto import tqdm - -from zenml import step -from zenml.enums import StrEnum -from zenml.client import Client - -experiment_tracker = Client().active_stack.experiment_tracker - - -@step -def full_evaluation_step( - evaluation_dataloader: DataLoader, - model: nn.Module, -) -> nn.Module: - """Data splitter step. - - This is an example of a data splitter step that splits the dataset into - training and dev subsets to be used for model training and evaluation. It - takes in a dataset as an step input artifact and returns the training and - dev subsets as two separate step output artifacts. - - Data splitter steps should have a deterministic behavior, i.e. they should - use a fixed random seed and always return the same split when called with - the same input dataset. This is to ensure reproducibility of your pipeline - runs. - - This step is parameterized using the `DataSplitterStepParameters` class, - which allows you to configure the step independently of the step code, - before running it in a pipeline. In this example, the step can be configured - to use a different random seed, change the split ratio, or control whether - to shuffle or stratify the split. See the documentation for more - information: - - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions - - Args: - params: Parameters for the data splitter step. - dataset: The dataset to split. - - Returns: - The resulting training and dev subsets. - """ - mlflow.pytorch.autolog() - model = BertForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=3) - optimizer = AdamW(model.parameters(), lr=5e-5) - num_epochs = 5 - num_training_steps = num_epochs * len(train_dataloader) - lr_scheduler = get_scheduler( - "linear", - optimizer=optimizer, - num_warmup_steps=0, - num_training_steps=num_training_steps, - ) - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") - model.to(device) - progress_bar = tqdm(range(num_training_steps)) - model.train() - for epoch in range(num_epochs): - for batch in train_dataloader: - batch = {k: v.to(device) for k, v in batch.items()} - outputs = model(**batch) - loss = outputs.loss - loss.backward() - - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() - progress_bar.update(1) - - loss, current = loss.item(), batch * len(X) - logger.info(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") - - return model \ No newline at end of file diff --git a/template/steps/training/{% if custom_training %}full_training.py{% endif %} b/template/steps/training/{% if custom_training %}full_training.py{% endif %} deleted file mode 100644 index 823cc80..0000000 --- a/template/steps/training/{% if custom_training %}full_training.py{% endif %} +++ /dev/null @@ -1,79 +0,0 @@ -# {% include 'template/license_header' %} - - -import torch -from torch import nn -from transformers import AdamW -from transformers import get_scheduler -from tqdm.auto import tqdm - -from zenml import step -from zenml.enums import StrEnum -from zenml.client import Client - -experiment_tracker = Client().active_stack.experiment_tracker - -@step -def full_training_step( - train_dataloader: DataLoader, - hf_pretrained_model: HFPretrainedModel, -) -> nn.Module: - """Data splitter step. - - This is an example of a data splitter step that splits the dataset into - training and dev subsets to be used for model training and evaluation. It - takes in a dataset as an step input artifact and returns the training and - dev subsets as two separate step output artifacts. - - Data splitter steps should have a deterministic behavior, i.e. they should - use a fixed random seed and always return the same split when called with - the same input dataset. This is to ensure reproducibility of your pipeline - runs. - - This step is parameterized using the `DataSplitterStepParameters` class, - which allows you to configure the step independently of the step code, - before running it in a pipeline. In this example, the step can be configured - to use a different random seed, change the split ratio, or control whether - to shuffle or stratify the split. See the documentation for more - information: - - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions - - Args: - params: Parameters for the data splitter step. - dataset: The dataset to split. - - Returns: - The resulting training and dev subsets. - """ - mlflow.pytorch.autolog() - model = BertForSequenceClassification.from_pretrained(hf_pretrained_model.value, num_labels=3) - optimizer = AdamW(model.parameters(), lr=5e-5) - num_epochs = 5 - num_training_steps = num_epochs * len(train_dataloader) - lr_scheduler = get_scheduler( - "linear", - optimizer=optimizer, - num_warmup_steps=0, - num_training_steps=num_training_steps, - ) - device = torch.device("cuda") if torch.cuda.is_available() else torch.device("cpu") - model.to(device) - progress_bar = tqdm(range(num_training_steps)) - model.train() - for epoch in range(num_epochs): - for batch in train_dataloader: - batch = {k: v.to(device) for k, v in batch.items()} - outputs = model(**batch) - loss = outputs.loss - loss.backward() - - optimizer.step() - lr_scheduler.step() - optimizer.zero_grad() - progress_bar.update(1) - - loss, current = loss.item(), batch * len(X) - logger.info(f"loss: {loss:>7f} [{current:>5d}/{size:>5d}]") - - return model \ No newline at end of file diff --git a/template/utils/misc.py b/template/utils/misc.py index 54872b0..24d7133 100644 --- a/template/utils/misc.py +++ b/template/utils/misc.py @@ -1,39 +1,52 @@ # {% include 'template/license_header' %} import numpy as np -from sklearn.metrics import accuracy_score, recall_score, precision_score, f1_score +from datasets import load_metric from zenml.enums import StrEnum -def compute_metrics(p): - pred, labels = p - pred = np.argmax(pred, axis=1) - - accuracy = accuracy_score(y_true=labels, y_pred=pred) - recall = recall_score(y_true=labels, y_pred=pred, average='micro') - precision = precision_score(y_true=labels, y_pred=pred, average='micro') - f1 = f1_score(y_true=labels, y_pred=pred, average='micro') - - return {"accuracy": accuracy, "precision": precision, "recall": recall, "f1": f1} - - - -def find_max_length(dataset): +def compute_metrics(eval_pred: tuple) -> dict[str, float]: + """Compute the metrics for the model. + + Args: + eval_pred: The evaluation prediction. + + Returns: + The metrics for the model. + """ + logits, labels = eval_pred + predictions = np.argmax(logits, axis=-1) + # calculate the mertic using the predicted and true value + accuracy = load_metric("accuracy").compute(predictions=predictions, references=labels) + f1 = load_metric("f1").compute(predictions=predictions, references=labels, average="weighted") + precision = load_metric("precision").compute(predictions=predictions, references=labels, average="weighted") + return {"accuracy": accuracy, "f1": f1, "precision": precision} + +def find_max_length(dataset: list[str]) -> int: + """Find the maximum length of the dataset. + + Args: + dataset: The dataset. + + Returns: + The maximum length of the dataset. + """ return len(max(dataset, key=lambda x: len(x.split())).split()) - class HFSentimentAnalysisDataset(StrEnum): """HuggingFace Sentiment Analysis datasets.""" financial_news = "zeroshot/twitter-financial-news-sentiment" - imbd_reviews = "mushroomsolutions/imdb_sentiment_3000_Test" - airline_reviews = "mattbit/tweet-sentiment-airlines" + imbd_reviews = "imdb" + airline_reviews = "Shayanvsf/US_Airline_Sentiment" class HFPretrainedModel(StrEnum): """HuggingFace Sentiment Analysis Model.""" bert = "bert-base-uncased" - gpt2 = "gpt2" + roberta = "roberta-base" + distilbert = "distilbert-base-cased" class HFPretrainedTokenizer(StrEnum): """HuggingFace Sentiment Analysis datasets.""" bert = "bert-base-uncased" - gpt2 = "gpt2" \ No newline at end of file + roberta = "roberta-base" + distilbert = "distilbert-base-cased" \ No newline at end of file From 95a7f4086e4cb02f45756fde0c1b0df10d475d3e Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:30:24 +0100 Subject: [PATCH 04/18] add deploy --- .scripts/format.sh | 14 ++++ copier.yml | 28 +++++-- template/config.py | 83 ------------------- template/config.yaml | 2 + template/gradio/app.py | 55 ++++++++---- template/gradio/requirements.txt | 12 +++ template/gradio/serve.yaml | 28 +++++++ template/pipelines/deploying.py | 83 +++++++++++++++++++ template/pipelines/promoting.py | 5 +- template/pipelines/training.py | 12 ++- template/steps/__init__.py | 16 ++-- template/steps/alerts/notify_on.py | 2 +- template/steps/dataset_loader/data_loader.py | 4 +- template/steps/deploying/save_model.py | 50 +++++++++++ ...e\" %}deploy_to_huggingface.py{% endif %}" | 41 +++++++++ ...l\" %}deploy_locally copy 2.py{% endif %}" | 53 ++++++++++++ ...kypilot\" %}deploy_skypilot.py{% endif %}" | 0 template/steps/promotion/__init__.py | 7 +- .../steps/promotion/promote_get_versions.py | 4 +- template/steps/registrer/__init__.py | 4 + ...{log_register.py => model_log_register.py} | 9 +- template/steps/training/model_trainer.py | 13 --- template/utils/misc.py | 27 ++++-- 23 files changed, 401 insertions(+), 151 deletions(-) create mode 100755 .scripts/format.sh delete mode 100644 template/config.py create mode 100644 template/gradio/requirements.txt create mode 100644 template/gradio/serve.yaml create mode 100644 template/pipelines/deploying.py create mode 100644 template/steps/deploying/save_model.py create mode 100644 "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" create mode 100644 "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" create mode 100644 "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_skypilot.py{% endif %}" create mode 100644 template/steps/registrer/__init__.py rename template/steps/registrer/{log_register.py => model_log_register.py} (95%) diff --git a/.scripts/format.sh b/.scripts/format.sh new file mode 100755 index 0000000..8113e92 --- /dev/null +++ b/.scripts/format.sh @@ -0,0 +1,14 @@ +#!/bin/sh -e +set -x + +SRC=${1:-"template tests .scripts"} + +export ZENML_DEBUG=1 +export ZENML_ANALYTICS_OPT_IN=false + +# autoflake replacement: removes unused imports and variables +ruff $SRC --select F401,F841 --fix --exclude "__init__.py" --isolated + +# sorts imports +ruff $SRC --select I --fix --ignore D +black $SRC diff --git a/copier.yml b/copier.yml index a02e5b1..230b666 100644 --- a/copier.yml +++ b/copier.yml @@ -58,6 +58,21 @@ target_environment: - production - staging default: staging +accelerator: + type: str + help: "The accelerator to use for training" + choices: + - gpu + - cpu + default: gpu +deployment_platform: + type: str + help: "The accelerator to use for training" + choices: + - local + - huggingface + - skypilot + default: local dataset: type: str help: "The name of the dataset to use from HuggingFace Datasets" @@ -65,14 +80,15 @@ dataset: - financial_news - airline_reviews - imbd_reviews - default: financial_news + default: imbd_reviews model: type: str help: "The name of the model to use from HuggingFace Models" choices: - - bert - - gpt2 - default: bert + - bert-base-uncased + - roberta-base + - distilbert-base-cased + default: roberta-base cloud_of_choice: type: str help: "Whether to use AWS cloud provider or GCP" @@ -92,10 +108,6 @@ zenml_server_url: type: str help: "The URL of the ZenML server [Optional]" default: "" -custom_training: - type: bool - help: "Whether to use custom training or not" - default: False # CONFIGURATION ------------------------- _templates_suffix: "" diff --git a/template/config.py b/template/config.py deleted file mode 100644 index 60d916e..0000000 --- a/template/config.py +++ /dev/null @@ -1,83 +0,0 @@ -# {% include 'template/license_header' %} - - -from artifacts.model_metadata import ModelMetadata -from pydantic import BaseConfig - -from zenml.config import DockerSettings -from utils.misc import ( - HFSentimentAnalysisDataset, - HFPretrainedModel, - HFPretrainedTokenizer, -) -from zenml.integrations.constants import ( -{%- if cloud_of_choice == 'aws' %} - SKYPILOT_AWS, - AWS, - S3, -{%- endif %} -{%- if cloud_of_choice == 'gcp' %} - SKYPILOT_GCP, - GCP, -{%- endif %} - HUGGINGFACE, - PYTORCH, - MLFLOW, - SLACK, - -) -from zenml.model_registries.base_model_registry import ModelVersionStage - -PIPELINE_SETTINGS = dict( - docker=DockerSettings( - required_integrations=[ - {%- if cloud_of_choice == 'aws' %} - SKYPILOT_AWS, - AWS, - S3, - {%- endif %} - {%- if cloud_of_choice == 'gcp' %} - SKYPILOT_GCP, - GCP, - {%- endif %} - HUGGINGFACE, - PYTORCH, - MLFLOW, - SLACK, - ], - requirements=[ - "accelerate", - ], - ) -) - -DEFAULT_PIPELINE_EXTRAS = dict( - notify_on_success={{notify_on_successes}}, - notify_on_failure={{notify_on_failures}} -) - -class MetaConfig(BaseConfig): -{%- if dataset == 'imbd_reviews' %} - dataset = HFSentimentAnalysisDataset.imbd_reviews -{%- endif %} -{%- if dataset == 'airline_reviews' %} - dataset = HFSentimentAnalysisDataset.airline_reviews -{%- else %} - dataset = HFSentimentAnalysisDataset.financial_news -{%- endif %} -{%- if model == 'gpt2' %} - tokenizer = HFPretrainedTokenizer.gpt2 - model = HFPretrainedModel.gpt2 -{%- else %} - tokenizer = HFPretrainedTokenizer.bert - model = HFPretrainedModel.bert -{%- endif %} - pipeline_name_training = "{{product_name}}_training" - mlflow_model_name = "{{product_name}}_model" -{%- if target_environment == 'production' %} - target_env = ModelVersionStage.PRODUCTION -{%- else %} - target_env = ModelVersionStage.STAGING -{%- endif %} - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### \ No newline at end of file diff --git a/template/config.yaml b/template/config.yaml index 39fa795..277fbe4 100644 --- a/template/config.yaml +++ b/template/config.yaml @@ -2,7 +2,9 @@ settings: docker: +{%- if accelerator == 'gpu' %} parent_image: 'huggingface/transformers-pytorch-gpu' +{%- endif %} required_integrations: {%- if cloud_of_choice == 'aws' %} - aws diff --git a/template/gradio/app.py b/template/gradio/app.py index 8ec5db1..4cc5441 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -1,19 +1,39 @@ import click -import gradio as gr -from transformers import AutoTokenizer, AutoModelForSequenceClassification import numpy as np +from transformers import AutoModelForSequenceClassification, AutoTokenizer + +import gradio as gr + @click.command() -@click.option('--tokenizer_name', default='roberta-base', help='Name of the tokenizer.') -@click.option('--model_name', default='bert', help='Name of the model.') -@click.option('--labels', default='Negative,Positive', help='Comma-separated list of labels.') -@click.option('--title', default='ZenML NLP Use-Case', help='Title of the Gradio interface.') -@click.option('--description', default='Tweets Analyzer', help='Description of the Gradio interface.') -@click.option('--interpretation', default='default', help='Interpretation mode for the Gradio interface.') -@click.option('--examples', default='bert,This airline sucks -_-', help='Comma-separated list of examples to show in the Gradio interface.') -def sentiment_analysis(tokenizer_name, model_name, labels, title, description, interpretation, examples): - labels = labels.split(',') - examples = [examples.split(',')] +@click.option("--tokenizer_name_or_path", default="roberta-base", help="Name or the path of the tokenizer.") +@click.option("--model_name_or_path", default="./gradio/model", help="Name or the path of the model.") +@click.option( + "--labels", default="Negative,Positive", help="Comma-separated list of labels." +) +@click.option( + "--title", default="ZenML NLP Use-Case", help="Title of the Gradio interface." +) +@click.option( + "--description", + default="Sentiment Analyzer", + help="Description of the Gradio interface.", +) +@click.option( + "--interpretation", + default="default", + help="Interpretation mode for the Gradio interface.", +) +@click.option( + "--examples", + default="bert,This airline sucks -_-", + help="Comma-separated list of examples to show in the Gradio interface.", +) +def sentiment_analysis( + tokenizer_name_or_path, model_name_or_path, labels, title, description, interpretation, examples +): + labels = labels.split(",") + examples = [examples.split(",")] def preprocess(text): new_text = [] @@ -28,8 +48,8 @@ def softmax(x): return e_x / e_x.sum(axis=0) def analyze_text(text): - tokenizer = AutoTokenizer.from_pretrained(tokenizer_name, do_lower_case=True) - model = AutoModelForSequenceClassification.from_pretrained(model_name) + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, do_lower_case=True) + model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path) text = preprocess(text) encoded_input = tokenizer(text, return_tensors="pt") @@ -47,10 +67,11 @@ def analyze_text(text): title=title, description=description, interpretation=interpretation, - examples=examples + examples=examples, ) demo.launch(share=True, debug=True) -if __name__ == '__main__': - sentiment_analysis() \ No newline at end of file + +if __name__ == "__main__": + sentiment_analysis() diff --git a/template/gradio/requirements.txt b/template/gradio/requirements.txt new file mode 100644 index 0000000..4ae46ad --- /dev/null +++ b/template/gradio/requirements.txt @@ -0,0 +1,12 @@ +nltk +torch +torchvision +torchaudio +gradio +datasets==2.12.0 +numpy==1.22.4 +pandas==1.5.3 +session_info==1.0.0 +scikit-learn==1.2.2 +transformers==4.28.1 +IPython==7.34.0 \ No newline at end of file diff --git a/template/gradio/serve.yaml b/template/gradio/serve.yaml new file mode 100644 index 0000000..d88b480 --- /dev/null +++ b/template/gradio/serve.yaml @@ -0,0 +1,28 @@ +# Task name (optional), used for display purposes. +name: nlp_use_case + +# Working directory (optional), synced to ~/sky_workdir on the remote cluster +# each time launch or exec is run with the yaml file. +# +# Commands in "setup" and "run" will be executed under it. +# +# If a .gitignore file (or a .git/info/exclude file) exists in the working +# directory, files and directories listed in it will be excluded from syncing. +workdir: ./gradio + +setup: | + echo "Begin setup." + pip install -r requirements.txt + echo "Setup complete." + +run: | + conda activate vllm + echo 'Starting vllm api server...' + python -u -m app.py \ + ----tokenizer_name $MODEL_NAME \ + --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \ + --tokenizer hf-internal-testing/llama-tokenizer 2>&1 | tee api_server.log & + echo 'Waiting for vllm api server to start...' + while ! `cat api_server.log | grep -q 'Uvicorn running on'`; do sleep 1; done + echo 'Starting gradio server...' + python vllm/examples/gradio_webserver.py \ No newline at end of file diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py new file mode 100644 index 0000000..5284e78 --- /dev/null +++ b/template/pipelines/deploying.py @@ -0,0 +1,83 @@ +# {% include 'template/license_header' %} + +from typing import Optional + +from steps import ( + notify_on_failure, + notify_on_success, +{%- if metric_compare_promotion %} + promote_get_metric, + promote_metric_compare_promoter, +{%- else %} + promote_latest, +{%- endif %} + promote_get_versions, +) +from zenml import get_pipeline_context +from zenml.logger import get_logger + +logger = get_logger(__name__) + +# Get experiment tracker +orchestrator = Client().active_stack.orchestrator + +# Check if orchestrator flavor is either default or skypilot +if orchestrator.flavor not in ["default"]: + raise RuntimeError( + "Your active stack needs to contain a default or skypilot orchestrator for " + "the deployment pipeline to work." + ) + +@pipeline( + on_failure=notify_on_failure, +) +def {{product_name}}_{{deployment}}_deployment( + labels: Optional[dict] = None, + title: Optional[str] = None, + description: Optional[str] = None, + model_name_or_path: Optional[str] = "./gardio/model" + tokenizer_name_or_path: Optional[str] = "{{model}}" +): + """ + Model deployment pipeline. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Link all the steps together by calling them and passing the output + # of one step as the input of the next step. + pipeline_extra = get_pipeline_context().extra + ########## Promotion stage ########## + save_model_locally( + mlflow_model_name=pipeline_extra["mlflow_model_name"], + stage=pipeline_extra["target_env"], + ) +{%- if deployment_platform == "local" %} + deploy_local( + model="{{model}}", + labels=labels, + title=title, + description=description, + model_name_or_path=model_name_or_path, + tokenizer_name_or_path=tokenizer_name_or_path, + ) +{%- endif %} +{%- if deployment_platform == "huggingface" %} + deploy_to_huggingface( + repo_name="{{project_name}}", + labels=labels, + title=title, + description=description, + ) +{%- endif %} +{%- if deployment_platform == "skypilot" %} + deploy_to_skypilot( + model="{{model}}", + labels=labels, + title=title, + description=description, + model_name_or_path=model_name_or_path, + tokenizer_name_or_path=tokenizer_name_or_path, + ) +{%- endif %} + + notify_on_success(after=[last_step_name]) + ### YOUR CODE ENDS HERE ### diff --git a/template/pipelines/promoting.py b/template/pipelines/promoting.py index c3bed5c..a443263 100644 --- a/template/pipelines/promoting.py +++ b/template/pipelines/promoting.py @@ -22,12 +22,9 @@ @pipeline( on_failure=notify_on_failure, ) -def {{product}}_promote_{{dataset}}(): +def {{product_name}}_promote_{{dataset}}(): """ Model promotion pipeline. - - - """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### # Link all the steps together by calling them and passing the output diff --git a/template/pipelines/training.py b/template/pipelines/training.py index b3f8adc..ecf83aa 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -10,7 +10,13 @@ tokenizer_loader, tokenization_step, model_trainer, - log_register, + model_log_register, +{%- if metric_compare_promotion %} + promote_get_metric, + promote_metric_compare_promoter, +{%- else %} + promote_latest, +{%- endif %} ) from zenml import pipeline, get_pipeline_context from zenml.logger import get_logger @@ -83,11 +89,11 @@ def {{product_name}}_training_{{dataset}}( ) ########## Log and Register stage ########## - log_register( + model_log_register( model=model, tokenizer=tokenizer, name="{{product_name}}_training_{{dataset}}", ) - notify_on_success(after=[log_register]) + notify_on_success(after=[model_log_register]) ### YOUR CODE ENDS HERE ### diff --git a/template/steps/__init__.py b/template/steps/__init__.py index c19d1a2..e86baa9 100644 --- a/template/steps/__init__.py +++ b/template/steps/__init__.py @@ -5,16 +5,20 @@ from .dataset_loader import ( data_loader, ) +from .promotion import ( +{%- if metric_compare_promotion %} + promote_get_metric, + promote_metric_compare_promoter, +{%- else %} + promote_latest, +{%- endif %} + promote_get_versions, +) +from .registrer import model_log_register from .tokenizer_loader import ( tokenizer_loader, ) from .tokenzation import ( tokenization_step, ) -from .inference import inference_get_current_version, inference_predict -from .promotion import ( - promote_latest, - promote_get_versions, -) from .training import model_trainer -from .registrer import log_register diff --git a/template/steps/alerts/notify_on.py b/template/steps/alerts/notify_on.py index 33ad394..d873f9a 100644 --- a/template/steps/alerts/notify_on.py +++ b/template/steps/alerts/notify_on.py @@ -54,4 +54,4 @@ def notify_on_success() -> None: """Notifies user on pipeline success.""" step_context = get_step_context() if alerter and step_context.pipeline_run.config.extra["notify_on_success"]: - alerter.post(message=build_message(status="succeeded")) \ No newline at end of file + alerter.post(message=build_message(status="succeeded")) diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index 1786ae5..18508ed 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -50,8 +50,8 @@ def data_loader( # Log the dataset and sample examples logger.info(dataset) - logger.info("Sample Example :", dataset["train"][0]) - logger.info("Sample Example :", dataset["train"][1]) + logger.info("Sample Example 1 :", dataset["train"][0]) + logger.info("Sample Example 2 :", dataset["train"][1]) ### YOUR CODE ENDS HERE ### return dataset \ No newline at end of file diff --git a/template/steps/deploying/save_model.py b/template/steps/deploying/save_model.py new file mode 100644 index 0000000..9d9e6c5 --- /dev/null +++ b/template/steps/deploying/save_model.py @@ -0,0 +1,50 @@ +# {% include 'template/license_header' %} + +from typing import Optional + +import mlflow +from transformers import ( + PreTrainedModel, + PreTrainedTokenizerBase, +) +from zenml import step +from zenml.client import Client +from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker +from zenml.logger import get_logger + +# Initialize logger +logger = get_logger(__name__) + +# Get experiment tracker +model_registry = Client().active_stack.model_registry + + +@step() +def save_model_locally( + mlflow_model_name: str, + stage: str, +): + """ + + + Args: + mlfow_model_name: The name of the model in MLFlow. + stage: The stage of the model in MLFlow. + + Returns: + The trained model and tokenizer. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + # Load model from MLFlow registry + loaded_model = model_registry.load_model_version( + name=mlflow_model_name, + version=stage, + ) + # Save the model and tokenizer locally + model_path = "./gradio/model" # replace with the actual path + tokenizer_path = "./gradio/tokenizer" # replace with the actual path + + # Save model locally + model = loaded_model["model"].save_pretrained(model_path) + tokenizer = loaded_model["tokenizer"].save_pretrained(tokenizer_path) + ### YOUR CODE ENDS HERE ### diff --git "a/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" new file mode 100644 index 0000000..88e758e --- /dev/null +++ "b/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" @@ -0,0 +1,41 @@ +# {% include 'template/license_header' %} + +from typing import Optional +from huggingface_hub import create_branch, login, HfApi + +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger + +# Initialize logger +logger = get_logger(__name__) + + +@step() +def deploy_to_huggingface( + repo_name: str, + labels: dict[str, str] = ["Negative", "Positive"], + title: str = "ZenML", + description: str = "ZenML NLP Use-Case", +): + """ + + Args: + mlfow_model_name: The name of the model in MLFlow. + stage: The stage of the model in MLFlow. + + Returns: + The trained model and tokenizer. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + lables = ",".join(labels) + secret = Client().get_secret("huggingface_creds") + huggingface_username = secret.secret_values["username"] + token = secret.secret_values["token"] + api = HfApi(token=token) + api.create_repo(repo_id=repo_name, repo_type="space", exist_ok=True) + space = api.upload_folder( + path="./gradio", repo_id=repo_name, repo_type="space" + ) + logger.info(f"Space created: {space}") + ### YOUR CODE ENDS HERE ### diff --git "a/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" new file mode 100644 index 0000000..085e90a --- /dev/null +++ "b/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" @@ -0,0 +1,53 @@ +# {% include 'template/license_header' %} + +from typing import Optional +import subprocess + +from transformers import ( + PreTrainedModel, + PreTrainedTokenizerBase, +) +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger + +# Initialize logger +logger = get_logger(__name__) + + +@step() +def deploy_locally( + labels: dict[str, str] = ["Negative", "Positive"], + title: str = "ZenML", + description: str = "ZenML NLP Use-Case", + model_name_or_path: str, + tokenizer_name_or_path: str, +): + """ + + Args: + mlfow_model_name: The name of the model in MLFlow. + stage: The stage of the model in MLFlow. + + Returns: + The trained model and tokenizer. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + lables = ",".join(labels) + command = ["python", "./gradio/app.py", "--tokenizer_name", tokenizer_name_or_path, "--model_name", model_name_or_path, + "--labels", lables, "--title", "ZenML", "--description", description, + "--interpretation", "default", "--examples", "This use-case is awesome!"] + + # Launch the script in a separate process + process = subprocess.Popen(command) + + # Print the process ID + print(f"Launched script with process ID: {process.pid}") + + return process.pid + + # Call the function to launch the script + pid = launch_script() + logger.info(f"Process ID: {pid}") + logger.info(f"To kill the process, run: kill -9 {pid}") + ### YOUR CODE ENDS HERE ### diff --git "a/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_skypilot.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_skypilot.py{% endif %}" new file mode 100644 index 0000000..e69de29 diff --git a/template/steps/promotion/__init__.py b/template/steps/promotion/__init__.py index aca59cf..a984e2f 100644 --- a/template/steps/promotion/__init__.py +++ b/template/steps/promotion/__init__.py @@ -1,5 +1,10 @@ # {% include 'template/license_header' %} -from .promote_latest import promote_latest from .promote_get_versions import promote_get_versions +{%- if metric_compare_promotion %} +from .promote_get_metric import promote_get_metric +from .promote_metric_compare_promoter import promote_metric_compare_promoter +{%- else %} +from .promote_latest import promote_latest +{%- endif %} \ No newline at end of file diff --git a/template/steps/promotion/promote_get_versions.py b/template/steps/promotion/promote_get_versions.py index c7042cb..7553cd9 100644 --- a/template/steps/promotion/promote_get_versions.py +++ b/template/steps/promotion/promote_get_versions.py @@ -2,9 +2,9 @@ from typing import Tuple -from typing_extensions import Annotated -from zenml import step, get_step_context +from typing_extensions import Annotated +from zenml import get_step_context, step from zenml.client import Client from zenml.logger import get_logger from zenml.model_registries.base_model_registry import ModelVersionStage diff --git a/template/steps/registrer/__init__.py b/template/steps/registrer/__init__.py new file mode 100644 index 0000000..ca4fcea --- /dev/null +++ b/template/steps/registrer/__init__.py @@ -0,0 +1,4 @@ +# {% include 'template/license_header' %} + + +from .model_log_register import model_log_register diff --git a/template/steps/registrer/log_register.py b/template/steps/registrer/model_log_register.py similarity index 95% rename from template/steps/registrer/log_register.py rename to template/steps/registrer/model_log_register.py index 41a195c..cd00270 100644 --- a/template/steps/registrer/log_register.py +++ b/template/steps/registrer/model_log_register.py @@ -1,12 +1,12 @@ # {% include 'template/license_header' %} from typing import Optional + import mlflow from transformers import ( PreTrainedModel, PreTrainedTokenizerBase, ) - from zenml import step from zenml.client import Client from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker @@ -27,8 +27,9 @@ "this example to work." ) + @step(experiment_tracker=experiment_tracker.name) -def model_register( +def model_log_register( model: PreTrainedModel, tokenizer: PreTrainedTokenizerBase, mlflow_model_name: Optional[str] = "model", @@ -41,7 +42,7 @@ def model_register( Model training steps should have caching disabled if they are not deterministic (i.e. if the model training involve some random processes like initializing - weights or shuffling data that are not controlled by setting a fixed random seed). + weights or shuffling data that are not controlled by setting a fixed random seed). Args: model: The model. @@ -61,4 +62,4 @@ def model_register( artifact_path=mlflow_model_name, registered_model_name=mlflow_model_name, ) - ### YOUR CODE ENDS HERE ### \ No newline at end of file + ### YOUR CODE ENDS HERE ### diff --git a/template/steps/training/model_trainer.py b/template/steps/training/model_trainer.py index f7b550e..3aea8f4 100644 --- a/template/steps/training/model_trainer.py +++ b/template/steps/training/model_trainer.py @@ -132,19 +132,6 @@ def model_trainer( # Train and evaluate the model trainer.train() trainer.evaluate() - - #{%- if log_at_trainer %} - # Log the model - #components = { - # "model": model, - # "tokenizer": tokenizer, - #} - #mlflow.transformers.log_model( - # transformers_model=components, - # artifact_path=mlflow_model_name, - # registered_model_name=mlflow_model_name, - #) - #{%- endif %} ### YOUR CODE ENDS HERE ### return model, tokenizer \ No newline at end of file diff --git a/template/utils/misc.py b/template/utils/misc.py index 24d7133..b4cba3a 100644 --- a/template/utils/misc.py +++ b/template/utils/misc.py @@ -2,26 +2,33 @@ import numpy as np from datasets import load_metric - from zenml.enums import StrEnum + def compute_metrics(eval_pred: tuple) -> dict[str, float]: """Compute the metrics for the model. - + Args: eval_pred: The evaluation prediction. - + Returns: The metrics for the model. """ logits, labels = eval_pred predictions = np.argmax(logits, axis=-1) # calculate the mertic using the predicted and true value - accuracy = load_metric("accuracy").compute(predictions=predictions, references=labels) - f1 = load_metric("f1").compute(predictions=predictions, references=labels, average="weighted") - precision = load_metric("precision").compute(predictions=predictions, references=labels, average="weighted") + accuracy = load_metric("accuracy").compute( + predictions=predictions, references=labels + ) + f1 = load_metric("f1").compute( + predictions=predictions, references=labels, average="weighted" + ) + precision = load_metric("precision").compute( + predictions=predictions, references=labels, average="weighted" + ) return {"accuracy": accuracy, "f1": f1, "precision": precision} + def find_max_length(dataset: list[str]) -> int: """Find the maximum length of the dataset. @@ -33,20 +40,26 @@ def find_max_length(dataset: list[str]) -> int: """ return len(max(dataset, key=lambda x: len(x.split())).split()) + class HFSentimentAnalysisDataset(StrEnum): """HuggingFace Sentiment Analysis datasets.""" + financial_news = "zeroshot/twitter-financial-news-sentiment" imbd_reviews = "imdb" airline_reviews = "Shayanvsf/US_Airline_Sentiment" + class HFPretrainedModel(StrEnum): """HuggingFace Sentiment Analysis Model.""" + bert = "bert-base-uncased" roberta = "roberta-base" distilbert = "distilbert-base-cased" + class HFPretrainedTokenizer(StrEnum): """HuggingFace Sentiment Analysis datasets.""" + bert = "bert-base-uncased" roberta = "roberta-base" - distilbert = "distilbert-base-cased" \ No newline at end of file + distilbert = "distilbert-base-cased" From 728cba6c4b507c4c5dac390dbdff74fd6e87c436 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Wed, 25 Oct 2023 22:44:05 +0100 Subject: [PATCH 05/18] fix dataset name --- template/gradio/app.py | 2 +- template/pipelines/deploying.py | 6 ++-- template/pipelines/promoting.py | 2 +- template/pipelines/training.py | 4 +-- template/run.py | 29 +++++++++++++++---- template/steps/dataset_loader/data_loader.py | 6 ++-- ... \"local\" %}deploy_locally.py{% endif %}" | 10 +++---- 7 files changed, 39 insertions(+), 20 deletions(-) rename "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" => "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" (83%) diff --git a/template/gradio/app.py b/template/gradio/app.py index 4cc5441..71c76c7 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -48,7 +48,7 @@ def softmax(x): return e_x / e_x.sum(axis=0) def analyze_text(text): - tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, do_lower_case=True) + tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path) model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path) text = preprocess(text) diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index 5284e78..a981a1a 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -31,12 +31,12 @@ @pipeline( on_failure=notify_on_failure, ) -def {{product_name}}_{{deployment}}_deployment( +def {{product_name}}_{{deployment_platform}}_deploying_pipeline( labels: Optional[dict] = None, title: Optional[str] = None, description: Optional[str] = None, - model_name_or_path: Optional[str] = "./gardio/model" - tokenizer_name_or_path: Optional[str] = "{{model}}" + model_name_or_path: Optional[str] = None, + tokenizer_name_or_path: Optional[str] = None, ): """ Model deployment pipeline. diff --git a/template/pipelines/promoting.py b/template/pipelines/promoting.py index a443263..75b34b9 100644 --- a/template/pipelines/promoting.py +++ b/template/pipelines/promoting.py @@ -22,7 +22,7 @@ @pipeline( on_failure=notify_on_failure, ) -def {{product_name}}_promote_{{dataset}}(): +def {{product_name}}_promote_pipeline(): """ Model promotion pipeline. """ diff --git a/template/pipelines/training.py b/template/pipelines/training.py index ecf83aa..fb653c4 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -26,7 +26,7 @@ @pipeline(on_failure=notify_on_failure) -def {{product_name}}_training_{{dataset}}( +def {{product_name}}_training_pipeline( lower_case: Optional[bool] = True, padding: Optional[str] = "max_length", max_seq_length: Optional[int] = 128, @@ -92,7 +92,7 @@ def {{product_name}}_training_{{dataset}}( model_log_register( model=model, tokenizer=tokenizer, - name="{{product_name}}_training_{{dataset}}", + name="{{product_name}}_model", ) notify_on_success(after=[model_log_register]) diff --git a/template/run.py b/template/run.py index 98bdbd9..6352f1a 100644 --- a/template/run.py +++ b/template/run.py @@ -84,14 +84,20 @@ type=click.FLOAT, help="Weight decay for training the model.", ) +@click.option( + "--promoting-pipeline", + is_flag=True, + default=False, + help="Whether to run the pipeline that promotes the model to {{target_environment}}.", +) def main( no_cache: bool = False, - seed: int = 42, - num_epochs: int = 5, - train_batch_size: int = 16, - eval_batch_size: int = 16, + num_epochs: int = 3, + train_batch_size: int = 8, + eval_batch_size: int = 8, learning_rate: float = 2e-5, weight_decay: float = 0.01, + promoting_pipeline: bool = False, ): """Main entry point for the pipeline execution. @@ -108,7 +114,12 @@ def main( # Run a pipeline with the required parameters. This executes # all steps in the pipeline in the correct order using the orchestrator # stack component that is configured in your active ZenML stack. - pipeline_args = {} + pipeline_args = { + "config_path":os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "config.yaml", + ) + } if no_cache: pipeline_args["enable_cache"] = False @@ -127,6 +138,14 @@ def main( {{product_name}}_training.with_options(**pipeline_args)(**run_args_train) logger.info("Training pipeline finished successfully!") + # Execute Promoting Pipeline + if promoting_pipeline: + run_args_promoting = {} + pipeline_args[ + "run_name" + ] = f"{{product_name}}_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + {{product_name}}_batch_inference.with_options(**pipeline_args)(**run_args_inference) + if __name__ == "__main__": main() diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index 18508ed..fc424bd 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -36,14 +36,14 @@ def data_loader( # Load dataset based on the dataset value {%- if dataset == 'financial_news' %} - dataset = load_dataset({{dataset}}) + dataset = load_dataset("zeroshot/twitter-financial-news-sentiment") {%- endif %} {%- if dataset == 'imbd_reviews' %} - dataset = load_dataset({{dataset}})["train"] + dataset = load_dataset("imdb")["train"] dataset = dataset.train_test_split(test_size=0.25, shuffle=shuffle) {%- endif %} {%- if dataset == 'airline_reviews' %} - dataset = load_dataset({{dataset}}) + dataset = load_dataset("Shayanvsf/US_Airline_Sentiment") dataset = dataset.rename_column("airline_sentiment", "label") dataset = dataset.remove_columns(["airline_sentiment_confidence","negativereason_confidence"]) {%- endif %} diff --git "a/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" similarity index 83% rename from "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" rename to "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" index 085e90a..ca2b331 100644 --- "a/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally copy 2.py{% endif %}" +++ "b/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" @@ -17,11 +17,11 @@ logger = get_logger(__name__) @step() def deploy_locally( - labels: dict[str, str] = ["Negative", "Positive"], - title: str = "ZenML", - description: str = "ZenML NLP Use-Case", - model_name_or_path: str, - tokenizer_name_or_path: str, + labels: Optional[dict[str, str]] = ["Negative", "Positive"], + title: Optional[str] = "ZenML", + description: Optional[str] = "ZenML NLP Use-Case", + model_name_or_path: str = "./gardio/model", + tokenizer_name_or_path: str = "./gradio/tokenizer", ): """ From 9dd6200d5224c5c326370c3c7ac05fadf01be774 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Wed, 25 Oct 2023 23:25:02 +0100 Subject: [PATCH 06/18] more fixes --- copier.yml | 2 +- template/pipelines/__init__.py | 2 + template/pipelines/deploying.py | 7 +- template/run.py | 75 +++++++++++++++---- template/steps/deploying/__init__.py | 13 ++++ template/steps/deploying/save_model.py | 2 +- ...ilot\" %}deploy_to_skypilot.py{% endif %}" | 0 ... \"local\" %}deploy_locally.py{% endif %}" | 0 8 files changed, 83 insertions(+), 18 deletions(-) create mode 100644 template/steps/deploying/__init__.py rename "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_skypilot.py{% endif %}" => "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_to_skypilot.py{% endif %}" (100%) rename "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" => "template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" (100%) diff --git a/copier.yml b/copier.yml index 230b666..18d0652 100644 --- a/copier.yml +++ b/copier.yml @@ -49,7 +49,7 @@ email: when: "{{ open_source_license != 'none' }}" product_name: type: str - help: The technical name of the data product you are building + help: The technical name of the data product you are building, Make sure it's one word and all lowercase (e.g. nlp_use_case) default: nlp_use_case target_environment: type: str diff --git a/template/pipelines/__init__.py b/template/pipelines/__init__.py index 67a5b51..3f2e7eb 100644 --- a/template/pipelines/__init__.py +++ b/template/pipelines/__init__.py @@ -2,3 +2,5 @@ from .training import {{product_name}}_training +from .promoting import {{product_name}}_promote_pipeline +from .deploying import {{product_name}}_{{deployment_platform}}_deploy_pipeline diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index a981a1a..2a7f2c7 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -31,7 +31,7 @@ @pipeline( on_failure=notify_on_failure, ) -def {{product_name}}_{{deployment_platform}}_deploying_pipeline( +def {{product_name}}_{{deployment_platform}}_deploy_pipeline( labels: Optional[dict] = None, title: Optional[str] = None, description: Optional[str] = None, @@ -46,7 +46,7 @@ def {{product_name}}_{{deployment_platform}}_deploying_pipeline( # of one step as the input of the next step. pipeline_extra = get_pipeline_context().extra ########## Promotion stage ########## - save_model_locally( + save_model_to_deploy( mlflow_model_name=pipeline_extra["mlflow_model_name"], stage=pipeline_extra["target_env"], ) @@ -59,6 +59,7 @@ def {{product_name}}_{{deployment_platform}}_deploying_pipeline( model_name_or_path=model_name_or_path, tokenizer_name_or_path=tokenizer_name_or_path, ) + last_step_name = "deploy_local" {%- endif %} {%- if deployment_platform == "huggingface" %} deploy_to_huggingface( @@ -67,6 +68,7 @@ def {{product_name}}_{{deployment_platform}}_deploying_pipeline( title=title, description=description, ) + last_step_name = "deploy_to_huggingface" {%- endif %} {%- if deployment_platform == "skypilot" %} deploy_to_skypilot( @@ -77,6 +79,7 @@ def {{product_name}}_{{deployment_platform}}_deploying_pipeline( model_name_or_path=model_name_or_path, tokenizer_name_or_path=tokenizer_name_or_path, ) + last_step_name = "deploy_to_skypilot" {%- endif %} notify_on_success(after=[last_step_name]) diff --git a/template/run.py b/template/run.py index 6352f1a..2d1f93f 100644 --- a/template/run.py +++ b/template/run.py @@ -3,7 +3,11 @@ from zenml.steps.external_artifact import ExternalArtifact from zenml.logger import get_logger -from pipelines import {{product_name}}_training +from pipelines import ( + {{product_name}}_training, + {{product_name}}_promote_pipeline, + {{product_name}}_{{deployment_platform}}_deploy_pipeline, +) import click from datetime import datetime as dt @@ -50,25 +54,19 @@ ) @click.option( "--num-epochs", - default=5, + default=3, type=click.INT, help="Number of epochs to train the model for.", ) -@click.option( - "--seed", - default=42, - type=click.INT, - help="Seed for the random number generator.", -) @click.option( "--train-batch-size", - default=16, + default=8, type=click.INT, help="Batch size for training the model.", ) @click.option( "--eval-batch-size", - default=16, + default=8, type=click.INT, help="Batch size for evaluating the model.", ) @@ -87,17 +85,52 @@ @click.option( "--promoting-pipeline", is_flag=True, - default=False, + default=True, help="Whether to run the pipeline that promotes the model to {{target_environment}}.", ) +@click.option( + "--deploying-pipeline", + is_flag=True, + default=True, + help="Whether to run the pipeline that deploys the model to {{deployment_platform}}.", +) +@click.option( + "--depployment-app-title", + default="Sentiment Analyzer", + type=click.STRING, + help="Title of the Gradio interface.", +) +@click.option( + "--depployment-app-description", + default="Sentiment Analyzer", + type=click.STRING, + help="Description of the Gradio interface.", +) +@click.option( + "--depployment-app-interpretation", + default="default", + type=click.STRING, + help="Interpretation mode for the Gradio interface.", +) +@click.option( + "--depployment-app-examples", + default="", + type=click.STRING, + help="Comma-separated list of examples to show in the Gradio interface.", +) def main( - no_cache: bool = False, + no_cache: bool = True, num_epochs: int = 3, train_batch_size: int = 8, eval_batch_size: int = 8, learning_rate: float = 2e-5, weight_decay: float = 0.01, - promoting_pipeline: bool = False, + promoting_pipeline: bool = True, + deploying_pipeline: bool = True, + depployment_app_title: str = "Sentiment Analyzer", + depployment_app_description: str = "Sentiment Analyzer", + depployment_app_interpretation: str = "default", + depployment_app_examples: str = "", ): """Main entry point for the pipeline execution. @@ -144,7 +177,21 @@ def main( pipeline_args[ "run_name" ] = f"{{product_name}}_promoting_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" - {{product_name}}_batch_inference.with_options(**pipeline_args)(**run_args_inference) + {{product_name}}_promote_pipeline.with_options(**pipeline_args)(**run_args_promoting) + logger.info("Promoting pipeline finished successfully!") + + if deploying_pipeline: + run_args_deploying = { + "title": depployment_app_title, + "description": depployment_app_description, + "interpretation": depployment_app_interpretation, + "examples": depployment_app_examples, + } + pipeline_args[ + "run_name" + ] = f"{{product_name}}_{{deployment_platform}}_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + {{product_name}}_{{deployment_platform}}_deploy_pipeline.with_options(**pipeline_args)(**run_args_deploying) + logger.info("Deploying pipeline finished successfully!") if __name__ == "__main__": diff --git a/template/steps/deploying/__init__.py b/template/steps/deploying/__init__.py new file mode 100644 index 0000000..fcbe9a5 --- /dev/null +++ b/template/steps/deploying/__init__.py @@ -0,0 +1,13 @@ +# {% include 'template/license_header' %} + + +from .save_model import save_model_to_deploy +{% if deployment_platform == "local" %} +from .deploy_locally import deploy_locally +{% endif %} +{% if deployment_platform == "huggingface" %} +from .deploy_to_huggingface import deploy_to_huggingface +{% endif %} +{% if deployment_platform == "skypilot" %} +from .deploy_to_skypilot import deploy_to_skypilot +{%- endif %} \ No newline at end of file diff --git a/template/steps/deploying/save_model.py b/template/steps/deploying/save_model.py index 9d9e6c5..7dd3dab 100644 --- a/template/steps/deploying/save_model.py +++ b/template/steps/deploying/save_model.py @@ -20,7 +20,7 @@ @step() -def save_model_locally( +def save_model_to_deploy( mlflow_model_name: str, stage: str, ): diff --git "a/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_skypilot.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_to_skypilot.py{% endif %}" similarity index 100% rename from "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_skypilot.py{% endif %}" rename to "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_to_skypilot.py{% endif %}" diff --git "a/template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" "b/template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" similarity index 100% rename from "template/steps/deploying/{% if deploy_platform == \"local\" %}deploy_locally.py{% endif %}" rename to "template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" From 157ec1b6e0ee9751b993b83c3bad58c6990d2ac5 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Thu, 26 Oct 2023 08:01:37 +0100 Subject: [PATCH 07/18] fix same naming and errors --- copier.yml | 2 +- template/gradio/app.py | 2 + template/pipelines/__init__.py | 2 +- template/pipelines/deploying.py | 22 ++++++----- template/pipelines/training.py | 8 ++-- template/run.py | 13 ++++--- template/steps/__init__.py | 15 +++++++- template/steps/deploying/__init__.py | 6 +-- ...\" %}huggingface_deployment.py{% endif %}" | 0 ...lot\" %}skypilot_deployment.py{% endif %}" | 0 ..."local\" %}local_deployment.py{% endif %}" | 37 ++++++++++++------- ...omotion %}promote_get_metric.py{% endif %} | 4 +- ...e_promotion %}promote_latest.py{% endif %} | 2 +- template/steps/registrer/__init__.py | 2 +- .../steps/registrer/model_log_register.py | 2 +- .../tokenizer_loader/tokenizer_loader.py | 14 +------ template/steps/tokenzation/tokenization.py | 8 ++-- template/steps/training/model_trainer.py | 9 +++-- template/utils/misc.py | 24 ------------ 19 files changed, 82 insertions(+), 90 deletions(-) rename "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" => "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" (100%) rename "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_to_skypilot.py{% endif %}" => "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}skypilot_deployment.py{% endif %}" (100%) rename "template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" => "template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" (62%) diff --git a/copier.yml b/copier.yml index 18d0652..bb570d6 100644 --- a/copier.yml +++ b/copier.yml @@ -80,7 +80,7 @@ dataset: - financial_news - airline_reviews - imbd_reviews - default: imbd_reviews + default: airline_reviews model: type: str help: "The name of the model to use from HuggingFace Models" diff --git a/template/gradio/app.py b/template/gradio/app.py index 71c76c7..0720eaf 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -1,3 +1,5 @@ +# {% include 'template/license_header' %} + import click import numpy as np from transformers import AutoModelForSequenceClassification, AutoTokenizer diff --git a/template/pipelines/__init__.py b/template/pipelines/__init__.py index 3f2e7eb..f2e66ac 100644 --- a/template/pipelines/__init__.py +++ b/template/pipelines/__init__.py @@ -1,6 +1,6 @@ # {% include 'template/license_header' %} -from .training import {{product_name}}_training +from .training import {{product_name}}_training_pipeline from .promoting import {{product_name}}_promote_pipeline from .deploying import {{product_name}}_{{deployment_platform}}_deploy_pipeline diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index 2a7f2c7..0809c82 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -5,16 +5,20 @@ from steps import ( notify_on_failure, notify_on_success, -{%- if metric_compare_promotion %} - promote_get_metric, - promote_metric_compare_promoter, -{%- else %} - promote_latest, + save_model_to_deploy, +{% if deployment_platform == "local" %} + deploy_locally, +{% endif %} +{% if deployment_platform == "huggingface" %} + deploy_to_huggingface, +{% endif %} +{% if deployment_platform == "skypilot" %} + deploy_to_skypilot, {%- endif %} - promote_get_versions, ) -from zenml import get_pipeline_context +from zenml import get_pipeline_context, pipeline from zenml.logger import get_logger +from zenml.client import Client logger = get_logger(__name__) @@ -22,7 +26,7 @@ orchestrator = Client().active_stack.orchestrator # Check if orchestrator flavor is either default or skypilot -if orchestrator.flavor not in ["default"]: +if orchestrator.flavor not in ["local", "vm_aws", "vm_gcp"]: raise RuntimeError( "Your active stack needs to contain a default or skypilot orchestrator for " "the deployment pipeline to work." @@ -51,7 +55,7 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( stage=pipeline_extra["target_env"], ) {%- if deployment_platform == "local" %} - deploy_local( + deploy_locally( model="{{model}}", labels=labels, title=title, diff --git a/template/pipelines/training.py b/template/pipelines/training.py index fb653c4..ab6fb16 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -10,7 +10,7 @@ tokenizer_loader, tokenization_step, model_trainer, - model_log_register, + register_model, {%- if metric_compare_promotion %} promote_get_metric, promote_metric_compare_promoter, @@ -89,11 +89,11 @@ def {{product_name}}_training_pipeline( ) ########## Log and Register stage ########## - model_log_register( + register_model( model=model, tokenizer=tokenizer, - name="{{product_name}}_model", + mlflow_model_name="{{product_name}}_model", ) - notify_on_success(after=[model_log_register]) + notify_on_success(after=["register_model"]) ### YOUR CODE ENDS HERE ### diff --git a/template/run.py b/template/run.py index 2d1f93f..f06d84e 100644 --- a/template/run.py +++ b/template/run.py @@ -1,15 +1,16 @@ # {% include 'template/license_header' %} +import os +import click +from datetime import datetime as dt -from zenml.steps.external_artifact import ExternalArtifact -from zenml.logger import get_logger from pipelines import ( - {{product_name}}_training, + {{product_name}}_training_pipeline, {{product_name}}_promote_pipeline, {{product_name}}_{{deployment_platform}}_deploy_pipeline, ) -import click -from datetime import datetime as dt +from zenml.logger import get_logger + logger = get_logger(__name__) @@ -168,7 +169,7 @@ def main( pipeline_args[ "run_name" ] = f"{{product_name}}_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" - {{product_name}}_training.with_options(**pipeline_args)(**run_args_train) + {{product_name}}_training_pipeline.with_options(**pipeline_args)(**run_args_train) logger.info("Training pipeline finished successfully!") # Execute Promoting Pipeline diff --git a/template/steps/__init__.py b/template/steps/__init__.py index e86baa9..e333285 100644 --- a/template/steps/__init__.py +++ b/template/steps/__init__.py @@ -14,7 +14,7 @@ {%- endif %} promote_get_versions, ) -from .registrer import model_log_register +from .registrer import register_model from .tokenizer_loader import ( tokenizer_loader, ) @@ -22,3 +22,16 @@ tokenization_step, ) from .training import model_trainer + +from .deploying import ( + save_model_to_deploy, +{% if deployment_platform == "local" %} + deploy_locally, +{% endif %} +{% if deployment_platform == "huggingface" %} + deploy_to_huggingface, +{% endif %} +{% if deployment_platform == "skypilot" %} + deploy_to_skypilot, +{%- endif %} +) diff --git a/template/steps/deploying/__init__.py b/template/steps/deploying/__init__.py index fcbe9a5..a6243ce 100644 --- a/template/steps/deploying/__init__.py +++ b/template/steps/deploying/__init__.py @@ -3,11 +3,11 @@ from .save_model import save_model_to_deploy {% if deployment_platform == "local" %} -from .deploy_locally import deploy_locally +from .local_deployment import deploy_locally {% endif %} {% if deployment_platform == "huggingface" %} -from .deploy_to_huggingface import deploy_to_huggingface +from .huggingface_deployment import deploy_to_huggingface {% endif %} {% if deployment_platform == "skypilot" %} -from .deploy_to_skypilot import deploy_to_skypilot +from .skypilot_deployment import deploy_to_skypilot {%- endif %} \ No newline at end of file diff --git "a/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" similarity index 100% rename from "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}deploy_to_huggingface.py{% endif %}" rename to "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" diff --git "a/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_to_skypilot.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}skypilot_deployment.py{% endif %}" similarity index 100% rename from "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}deploy_to_skypilot.py{% endif %}" rename to "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}skypilot_deployment.py{% endif %}" diff --git "a/template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" "b/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" similarity index 62% rename from "template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" rename to "template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" index ca2b331..76d0998 100644 --- "a/template/steps/deploying/{% if deployment_platform == \"local\" %}deploy_locally.py{% endif %}" +++ "b/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" @@ -3,10 +3,6 @@ from typing import Optional import subprocess -from transformers import ( - PreTrainedModel, - PreTrainedTokenizerBase, -) from zenml import step from zenml.client import Client from zenml.logger import get_logger @@ -33,21 +29,34 @@ def deploy_locally( The trained model and tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - lables = ",".join(labels) - command = ["python", "./gradio/app.py", "--tokenizer_name", tokenizer_name_or_path, "--model_name", model_name_or_path, - "--labels", lables, "--title", "ZenML", "--description", description, - "--interpretation", "default", "--examples", "This use-case is awesome!"] + def start_gradio_app(command: list[str]) -> int: + """ + Start the Gradio app in a separate process. + + Args: + command: The command to start the Gradio app. + + Returns: + The process ID of the Gradio app. + """ + # Define the command to start the Gradio app + command = ["python", "app.py"] - # Launch the script in a separate process - process = subprocess.Popen(command) + # Start the Gradio app in a separate process + process = subprocess.Popen(command) - # Print the process ID - print(f"Launched script with process ID: {process.pid}") + # Print the process ID + logger.info(f"Started Gradio app with process ID: {process.pid}") - return process.pid + return process.pid + + lables = ",".join(labels) + command = ["python", "./gradio/app.py", "--tokenizer_name", tokenizer_name_or_path, "--model_name", model_name_or_path, + "--labels", lables, "--title", title, "--description", description, + "--interpretation", "default", "--examples", "This use-case is awesome!"] # Call the function to launch the script - pid = launch_script() + pid = start_gradio_app(command) logger.info(f"Process ID: {pid}") logger.info(f"To kill the process, run: kill -9 {pid}") ### YOUR CODE ENDS HERE ### diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} index 9e0b13c..297eddf 100644 --- a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} @@ -3,9 +3,7 @@ from typing_extensions import Annotated -import pandas as pd -from sklearn.metrics import accuracy_score -from zenml import step +from zenml import get_step_context, step from zenml.client import Client from zenml.logger import get_logger from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker diff --git a/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} b/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} index 85aab9f..7c6014f 100644 --- a/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} +++ b/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} @@ -1,7 +1,7 @@ # {% include 'template/license_header' %} -from zenml import step +from zenml import get_step_context, step from zenml.client import Client from zenml.logger import get_logger from zenml.model_registries.base_model_registry import ModelVersionStage diff --git a/template/steps/registrer/__init__.py b/template/steps/registrer/__init__.py index ca4fcea..5f14b15 100644 --- a/template/steps/registrer/__init__.py +++ b/template/steps/registrer/__init__.py @@ -1,4 +1,4 @@ # {% include 'template/license_header' %} -from .model_log_register import model_log_register +from .model_log_register import register_model diff --git a/template/steps/registrer/model_log_register.py b/template/steps/registrer/model_log_register.py index cd00270..66adede 100644 --- a/template/steps/registrer/model_log_register.py +++ b/template/steps/registrer/model_log_register.py @@ -29,7 +29,7 @@ @step(experiment_tracker=experiment_tracker.name) -def model_log_register( +def register_model( model: PreTrainedModel, tokenizer: PreTrainedTokenizerBase, mlflow_model_name: Optional[str] = "model", diff --git a/template/steps/tokenizer_loader/tokenizer_loader.py b/template/steps/tokenizer_loader/tokenizer_loader.py index 71b1aba..0ec117e 100644 --- a/template/steps/tokenizer_loader/tokenizer_loader.py +++ b/template/steps/tokenizer_loader/tokenizer_loader.py @@ -36,21 +36,9 @@ def tokenizer_loader( The initialized tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - {%- if model == 'bert' %} tokenizer = AutoTokenizer.from_pretrained( - "bert-base-uncased", do_lower_case=lower_case + "{{model}}", do_lower_case=lower_case ) - {%- endif %} - {%- if model == 'roberta' %} - tokenizer = AutoTokenizer.from_pretrained( - "roberta-base", do_lower_case=lower_case - ) - {%- endif %} - {%- if model == 'distilbert' %} - tokenizer = AutoTokenizer.from_pretrained( - "distilbert-base-cased", do_lower_case=lower_case - ) - {%- endif %} ### YOUR CODE ENDS HERE ### return tokenizer diff --git a/template/steps/tokenzation/tokenization.py b/template/steps/tokenzation/tokenization.py index 11a47d4..bf79485 100644 --- a/template/steps/tokenzation/tokenization.py +++ b/template/steps/tokenzation/tokenization.py @@ -11,12 +11,12 @@ @step def tokenization_step( - padding: str, + tokenizer: PreTrainedTokenizerBase, + dataset: DatasetDict, + padding: str = "max_length", max_seq_length: int = 512, text_column: str = "text", label_column: str = "label", - tokenizer: PreTrainedTokenizerBase, - dataset: DatasetDict, ) -> Annotated[DatasetDict, "tokenized_data"]: """ Tokenization step. @@ -69,7 +69,7 @@ def preprocess_function(examples): # Tokenize the examples with padding, truncation, and a specified maximum length result = tokenizer( examples[text_column], - padding="max_length", + padding=padding, truncation=True, max_length=max_length or max_seq_length, ) diff --git a/template/steps/training/model_trainer.py b/template/steps/training/model_trainer.py index 3aea8f4..e55c5bf 100644 --- a/template/steps/training/model_trainer.py +++ b/template/steps/training/model_trainer.py @@ -17,8 +17,7 @@ from zenml.client import Client from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker from zenml.logger import get_logger - -from template.utils.misc import compute_metrics +from utils.misc import compute_metrics # Initialize logger logger = get_logger(__name__) @@ -89,7 +88,7 @@ def model_trainer( data_collator = DataCollatorWithPadding(tokenizer=tokenizer) # Set the number of labels - num_labels = num_labels or len(train_dataset.unique("labels")) + num_labels = len(train_dataset.unique("labels")) or num_labels # Set the logging steps logging_steps = len(train_dataset) // train_batch_size @@ -104,6 +103,8 @@ def model_trainer( weight_decay=weight_decay, evaluation_strategy='steps', save_strategy='steps', + save_steps=1000, + eval_steps=200, logging_steps=logging_steps, save_total_limit=5, report_to="mlflow", @@ -113,7 +114,7 @@ def model_trainer( # Load the model model = AutoModelForSequenceClassification.from_pretrained( - {{model}}, num_labels=num_labels + "{{model}}", num_labels=num_labels ) # Enable autologging diff --git a/template/utils/misc.py b/template/utils/misc.py index b4cba3a..14abde8 100644 --- a/template/utils/misc.py +++ b/template/utils/misc.py @@ -39,27 +39,3 @@ def find_max_length(dataset: list[str]) -> int: The maximum length of the dataset. """ return len(max(dataset, key=lambda x: len(x.split())).split()) - - -class HFSentimentAnalysisDataset(StrEnum): - """HuggingFace Sentiment Analysis datasets.""" - - financial_news = "zeroshot/twitter-financial-news-sentiment" - imbd_reviews = "imdb" - airline_reviews = "Shayanvsf/US_Airline_Sentiment" - - -class HFPretrainedModel(StrEnum): - """HuggingFace Sentiment Analysis Model.""" - - bert = "bert-base-uncased" - roberta = "roberta-base" - distilbert = "distilbert-base-cased" - - -class HFPretrainedTokenizer(StrEnum): - """HuggingFace Sentiment Analysis datasets.""" - - bert = "bert-base-uncased" - roberta = "roberta-base" - distilbert = "distilbert-base-cased" From bb78ab0cc6f72160b3fc23fcc5726d6739044c3e Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:54:14 +0100 Subject: [PATCH 08/18] fix dependencies and paarameter --- template/gradio/app.py | 2 +- template/pipelines/deploying.py | 6 ++++++ template/requirements.txt | 2 ++ template/run.py | 4 ++-- template/steps/dataset_loader/data_loader.py | 4 ++-- ... metric_compare_promotion %}promote_latest.py{% endif %} | 2 +- 6 files changed, 14 insertions(+), 6 deletions(-) diff --git a/template/gradio/app.py b/template/gradio/app.py index 0720eaf..30548cd 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -28,7 +28,7 @@ ) @click.option( "--examples", - default="bert,This airline sucks -_-", + default="This is an awesome journey, I love it!", help="Comma-separated list of examples to show in the Gradio interface.", ) def sentiment_analysis( diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index 0809c82..2bc9c19 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -41,6 +41,8 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( description: Optional[str] = None, model_name_or_path: Optional[str] = None, tokenizer_name_or_path: Optional[str] = None, + interpretation: Optional[str] = None, + example: Optional[str] = None, ): """ Model deployment pipeline. @@ -60,6 +62,8 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( labels=labels, title=title, description=description, + interpretation=interpretation, + example=example, model_name_or_path=model_name_or_path, tokenizer_name_or_path=tokenizer_name_or_path, ) @@ -80,6 +84,8 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( labels=labels, title=title, description=description, + interpretation=interpretation, + example=example, model_name_or_path=model_name_or_path, tokenizer_name_or_path=tokenizer_name_or_path, ) diff --git a/template/requirements.txt b/template/requirements.txt index 75c1620..52cc7f6 100644 --- a/template/requirements.txt +++ b/template/requirements.txt @@ -1 +1,3 @@ +torchvision +accelerate zenml[server] diff --git a/template/run.py b/template/run.py index f06d84e..5433022 100644 --- a/template/run.py +++ b/template/run.py @@ -131,7 +131,7 @@ def main( depployment_app_title: str = "Sentiment Analyzer", depployment_app_description: str = "Sentiment Analyzer", depployment_app_interpretation: str = "default", - depployment_app_examples: str = "", + depployment_app_example: str = "", ): """Main entry point for the pipeline execution. @@ -186,7 +186,7 @@ def main( "title": depployment_app_title, "description": depployment_app_description, "interpretation": depployment_app_interpretation, - "examples": depployment_app_examples, + "example": depployment_app_example, } pipeline_args[ "run_name" diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index fc424bd..ecd08be 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -50,8 +50,8 @@ def data_loader( # Log the dataset and sample examples logger.info(dataset) - logger.info("Sample Example 1 :", dataset["train"][0]) - logger.info("Sample Example 2 :", dataset["train"][1]) + logger.info(f"Sample Example 1 : {dataset['train'][0]['text']} with label {dataset['train'][0]['label']}") + logger.info(f"Sample Example 1 : {dataset['train'][1]['text']} with label {dataset['train'][1]['label']}") ### YOUR CODE ENDS HERE ### return dataset \ No newline at end of file diff --git a/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} b/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} index 7c6014f..7c335b8 100644 --- a/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} +++ b/template/steps/promotion/{% if not metric_compare_promotion %}promote_latest.py{% endif %} @@ -37,7 +37,7 @@ def promote_latest(latest_version:str, current_version:str): model_registry.update_model_version( name=pipeline_extra["mlflow_model_name"], version=latest_version, - stage=pipeline_extra["target_env"], + stage=ModelVersionStage(pipeline_extra["target_env"]), metadata={}, ) promoted_version = latest_version From facd30ab2f975227bce9bf28f0a8444bede3ea0c Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Thu, 26 Oct 2023 14:35:24 +0100 Subject: [PATCH 09/18] lower default epoch number and fix path for deploy --- template/Makefile | 13 +++++++++- template/pipelines/deploying.py | 15 ++++++------ template/requirements.txt | 1 + template/run.py | 4 ++-- template/steps/dataset_loader/data_loader.py | 3 ++- template/steps/deploying/save_model.py | 24 ++++++++++--------- ...\" %}huggingface_deployment.py{% endif %}" | 4 ++-- ..."local\" %}local_deployment.py{% endif %}" | 21 +++++++++------- .../steps/promotion/promote_get_versions.py | 2 +- .../steps/registrer/model_log_register.py | 1 + 10 files changed, 54 insertions(+), 34 deletions(-) diff --git a/template/Makefile b/template/Makefile index 4097971..f12a621 100644 --- a/template/Makefile +++ b/template/Makefile @@ -11,4 +11,15 @@ setup: pip install -r requirements.txt zenml integration install pytorch mlflow s3 gcp aws slack transformers -y -install-stack: +install-local-stack: + @echo "Specify stack name [$(stack_name)]: " && read input && [ -n "$$input" ] && stack_name="$$input" || stack_name="$(stack_name)" && \ + zenml experiment-tracker register -f mlflow mlflow_local_$${stack_name} && \ + zenml model-registry register -f mlflow mlflow_local_$${stack_name} && \ + zenml model-deployer register -f mlflow mlflow_local_$${stack_name} && \ + zenml stack register -a default -o default -r mlflow_local_$${stack_name} \ + -d mlflow_local_$${stack_name} -e mlflow_local_$${stack_name} $${stack_name} && \ + zenml stack set $${stack_name} && \ + zenml stack up + +install-remote-stack: + diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index 2bc9c19..844fe16 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -1,6 +1,6 @@ # {% include 'template/license_header' %} -from typing import Optional +from typing import Optional, List from steps import ( notify_on_failure, @@ -36,11 +36,11 @@ on_failure=notify_on_failure, ) def {{product_name}}_{{deployment_platform}}_deploy_pipeline( - labels: Optional[dict] = None, + labels: Optional[List[str]] = ["Negative", "Positive"], title: Optional[str] = None, description: Optional[str] = None, - model_name_or_path: Optional[str] = None, - tokenizer_name_or_path: Optional[str] = None, + model_name_or_path: Optional[str] = "gardio/model", + tokenizer_name_or_path: Optional[str] = "gradio/tokenizer", interpretation: Optional[str] = None, example: Optional[str] = None, ): @@ -58,7 +58,6 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( ) {%- if deployment_platform == "local" %} deploy_locally( - model="{{model}}", labels=labels, title=title, description=description, @@ -66,8 +65,9 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( example=example, model_name_or_path=model_name_or_path, tokenizer_name_or_path=tokenizer_name_or_path, + after=["save_model_to_deploy"], ) - last_step_name = "deploy_local" + last_step_name = "deploy_locally" {%- endif %} {%- if deployment_platform == "huggingface" %} deploy_to_huggingface( @@ -75,12 +75,12 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( labels=labels, title=title, description=description, + after=["save_model_to_deploy"], ) last_step_name = "deploy_to_huggingface" {%- endif %} {%- if deployment_platform == "skypilot" %} deploy_to_skypilot( - model="{{model}}", labels=labels, title=title, description=description, @@ -88,6 +88,7 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( example=example, model_name_or_path=model_name_or_path, tokenizer_name_or_path=tokenizer_name_or_path, + after=["save_model_to_deploy"], ) last_step_name = "deploy_to_skypilot" {%- endif %} diff --git a/template/requirements.txt b/template/requirements.txt index 52cc7f6..0087b4d 100644 --- a/template/requirements.txt +++ b/template/requirements.txt @@ -1,3 +1,4 @@ torchvision accelerate +gradio zenml[server] diff --git a/template/run.py b/template/run.py index 5433022..2eb87ae 100644 --- a/template/run.py +++ b/template/run.py @@ -55,7 +55,7 @@ ) @click.option( "--num-epochs", - default=3, + default=1, type=click.INT, help="Number of epochs to train the model for.", ) @@ -114,7 +114,7 @@ help="Interpretation mode for the Gradio interface.", ) @click.option( - "--depployment-app-examples", + "--depployment-app-example", default="", type=click.STRING, help="Comma-separated list of examples to show in the Gradio interface.", diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index ecd08be..413e803 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -32,7 +32,7 @@ def data_loader( The loaded dataset artifact. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - logger.info(f"Loaded dataset {{dataset}}") + logger.info(f"Loading dataset {{dataset}}... ") # Load dataset based on the dataset value {%- if dataset == 'financial_news' %} @@ -52,6 +52,7 @@ def data_loader( logger.info(dataset) logger.info(f"Sample Example 1 : {dataset['train'][0]['text']} with label {dataset['train'][0]['label']}") logger.info(f"Sample Example 1 : {dataset['train'][1]['text']} with label {dataset['train'][1]['label']}") + logger.info(f" Dataset Loaded Successfully") ### YOUR CODE ENDS HERE ### return dataset \ No newline at end of file diff --git a/template/steps/deploying/save_model.py b/template/steps/deploying/save_model.py index 7dd3dab..cd273d7 100644 --- a/template/steps/deploying/save_model.py +++ b/template/steps/deploying/save_model.py @@ -3,13 +3,9 @@ from typing import Optional import mlflow -from transformers import ( - PreTrainedModel, - PreTrainedTokenizerBase, -) from zenml import step from zenml.client import Client -from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker +from zenml.model_registries.base_model_registry import ModelVersionStage from zenml.logger import get_logger # Initialize logger @@ -25,8 +21,6 @@ def save_model_to_deploy( stage: str, ): """ - - Args: mlfow_model_name: The name of the model in MLFlow. stage: The stage of the model in MLFlow. @@ -35,16 +29,24 @@ def save_model_to_deploy( The trained model and tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + logger.info(f" Loading latest version of model {mlflow_model_name} for stage {stage}...") + # Load model from MLFlow registry + model_version = model_registry.get_latest_model_version( + name=mlflow_model_name, + stage=ModelVersionStage(stage), + ).version # Load model from MLFlow registry - loaded_model = model_registry.load_model_version( + model_version = model_registry.get_model_version( name=mlflow_model_name, - version=stage, + version=model_version, ) + transformer_model = mlflow.transformers.load_model(model_version.model_source_uri) # Save the model and tokenizer locally model_path = "./gradio/model" # replace with the actual path tokenizer_path = "./gradio/tokenizer" # replace with the actual path # Save model locally - model = loaded_model["model"].save_pretrained(model_path) - tokenizer = loaded_model["tokenizer"].save_pretrained(tokenizer_path) + transformer_model.model.save_pretrained(model_path) + transformer_model.tokenizer.save_pretrained(tokenizer_path) + logger.info(f" Model and tokenizer saved to {model_path} and {tokenizer_path} respectively.") ### YOUR CODE ENDS HERE ### diff --git "a/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" index 88e758e..eff5f20 100644 --- "a/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" +++ "b/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" @@ -1,6 +1,6 @@ # {% include 'template/license_header' %} -from typing import Optional +from typing import Optional, List from huggingface_hub import create_branch, login, HfApi from zenml import step @@ -14,7 +14,7 @@ logger = get_logger(__name__) @step() def deploy_to_huggingface( repo_name: str, - labels: dict[str, str] = ["Negative", "Positive"], + labels: Optional[List[str]] = ["Negative", "Positive"], title: str = "ZenML", description: str = "ZenML NLP Use-Case", ): diff --git "a/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" "b/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" index 76d0998..d05dbfe 100644 --- "a/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" +++ "b/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" @@ -1,6 +1,6 @@ # {% include 'template/license_header' %} -from typing import Optional +from typing import Optional, List import subprocess from zenml import step @@ -13,11 +13,13 @@ logger = get_logger(__name__) @step() def deploy_locally( - labels: Optional[dict[str, str]] = ["Negative", "Positive"], + model_name_or_path: str, + tokenizer_name_or_path: str, + labels: List[str], title: Optional[str] = "ZenML", description: Optional[str] = "ZenML NLP Use-Case", - model_name_or_path: str = "./gardio/model", - tokenizer_name_or_path: str = "./gradio/tokenizer", + interpretation: Optional[str] = "default", + example: Optional[str] = "This use-case is awesome!", ): """ @@ -39,9 +41,6 @@ def deploy_locally( Returns: The process ID of the Gradio app. """ - # Define the command to start the Gradio app - command = ["python", "app.py"] - # Start the Gradio app in a separate process process = subprocess.Popen(command) @@ -51,9 +50,13 @@ def deploy_locally( return process.pid lables = ",".join(labels) - command = ["python", "./gradio/app.py", "--tokenizer_name", tokenizer_name_or_path, "--model_name", model_name_or_path, + # Construct the path to the app.py file + app_path = str(os.path.join(Client().root, "gradio", "app.py")) + model_path = str(os.path.join(Client().root, model_name_or_path)) + tokenizer_path = str(os.path.join(Client().root, tokenizer_name_or_path)) + command = ["python", app_path, "--tokenizer_name_or_path", tokenizer_path, "--model_name_or_path", model_path, "--labels", lables, "--title", title, "--description", description, - "--interpretation", "default", "--examples", "This use-case is awesome!"] + "--interpretation", interpretation, "--examples", example] # Call the function to launch the script pid = start_gradio_app(command) diff --git a/template/steps/promotion/promote_get_versions.py b/template/steps/promotion/promote_get_versions.py index 7553cd9..96dc344 100644 --- a/template/steps/promotion/promote_get_versions.py +++ b/template/steps/promotion/promote_get_versions.py @@ -50,4 +50,4 @@ def promote_get_versions() -> ( logger.info("No currently promoted model version found.") ### YOUR CODE ENDS HERE ### - return current_version, current_version + return latest_versions, current_version diff --git a/template/steps/registrer/model_log_register.py b/template/steps/registrer/model_log_register.py index 66adede..97c150d 100644 --- a/template/steps/registrer/model_log_register.py +++ b/template/steps/registrer/model_log_register.py @@ -61,5 +61,6 @@ def register_model( transformers_model=components, artifact_path=mlflow_model_name, registered_model_name=mlflow_model_name, + task="text-classification", ) ### YOUR CODE ENDS HERE ### From 9643e008ab34a78f4e889f4e68f8fdc7f3402d93 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Thu, 26 Oct 2023 22:07:42 +0100 Subject: [PATCH 10/18] fix path --- template/gradio/app.py | 34 ++++++++++++++++--- template/pipelines/deploying.py | 4 +-- ..."local\" %}local_deployment.py{% endif %}" | 5 ++- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/template/gradio/app.py b/template/gradio/app.py index 30548cd..be81228 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -1,15 +1,35 @@ -# {% include 'template/license_header' %} +# Apache Software License 2.0 +# +# Copyright (c) ZenML GmbH 2023. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# import click import numpy as np +import os from transformers import AutoModelForSequenceClassification, AutoTokenizer +from os.path import dirname import gradio as gr +from zenml.logger import get_logger +# Initialize logger +logger = get_logger(__name__) @click.command() -@click.option("--tokenizer_name_or_path", default="roberta-base", help="Name or the path of the tokenizer.") -@click.option("--model_name_or_path", default="./gradio/model", help="Name or the path of the model.") +@click.option("--tokenizer_name_or_path", default="tokenizer", help="Name or the path of the tokenizer.") +@click.option("--model_name_or_path", default="model", help="Name or the path of the model.") @click.option( "--labels", default="Negative,Positive", help="Comma-separated list of labels." ) @@ -50,8 +70,12 @@ def softmax(x): return e_x / e_x.sum(axis=0) def analyze_text(text): - tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path) - model = AutoModelForSequenceClassification.from_pretrained(model_name_or_path) + model_path = f"{dirname(__file__)}/{tokenizer_name_or_path}/" + logger.info(f"Loading model from {model_path}") + tokenizer_path = f"{dirname(__file__)}/{model_name_or_path}/" + logger.info(f"Loading tokenizer from {tokenizer_path}") + tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + model = AutoModelForSequenceClassification.from_pretrained(model_path) text = preprocess(text) encoded_input = tokenizer(text, return_tensors="pt") diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index 844fe16..c5d6ef5 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -39,8 +39,8 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( labels: Optional[List[str]] = ["Negative", "Positive"], title: Optional[str] = None, description: Optional[str] = None, - model_name_or_path: Optional[str] = "gardio/model", - tokenizer_name_or_path: Optional[str] = "gradio/tokenizer", + model_name_or_path: Optional[str] = "model", + tokenizer_name_or_path: Optional[str] = "tokenizer", interpretation: Optional[str] = None, example: Optional[str] = None, ): diff --git "a/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" "b/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" index d05dbfe..0d48f37 100644 --- "a/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" +++ "b/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" @@ -1,6 +1,7 @@ # {% include 'template/license_header' %} from typing import Optional, List +import os import subprocess from zenml import step @@ -52,12 +53,10 @@ def deploy_locally( lables = ",".join(labels) # Construct the path to the app.py file app_path = str(os.path.join(Client().root, "gradio", "app.py")) - model_path = str(os.path.join(Client().root, model_name_or_path)) - tokenizer_path = str(os.path.join(Client().root, tokenizer_name_or_path)) command = ["python", app_path, "--tokenizer_name_or_path", tokenizer_path, "--model_name_or_path", model_path, "--labels", lables, "--title", title, "--description", description, "--interpretation", interpretation, "--examples", example] - + logger.info(f"Command: {command}") # Call the function to launch the script pid = start_gradio_app(command) logger.info(f"Process ID: {pid}") From 888bc9445cadbefb558a602b2563a3ff9dfe166e Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Fri, 27 Oct 2023 02:12:45 +0100 Subject: [PATCH 11/18] finalize deployment --- copier.yml | 24 ++++++---- template/gradio/app.py | 14 ++---- template/gradio/serve.yaml | 22 +++++---- template/pipelines/__init__.py | 2 +- template/pipelines/deploying.py | 47 ++++++++++--------- template/run.py | 9 ++-- template/steps/__init__.py | 6 +-- template/steps/deploying/__init__.py | 6 +-- ...y_locally %}local_deployment.py{% endif %} | 18 +++++-- ...lot\" %}skypilot_deployment.py{% endif %}" | 0 ...ace %}huggingface_deployment.py{% endif %} | 18 ++++--- ...ypilot %}skypilot_deployment.py{% endif %} | 40 ++++++++++++++++ 12 files changed, 136 insertions(+), 70 deletions(-) rename "template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" => template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} (71%) delete mode 100644 "template/steps/deploying/{% if deploy_platform == \"skypilot\" %}skypilot_deployment.py{% endif %}" rename "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" => template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} (53%) create mode 100644 template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} diff --git a/copier.yml b/copier.yml index bb570d6..54a6521 100644 --- a/copier.yml +++ b/copier.yml @@ -65,14 +65,18 @@ accelerator: - gpu - cpu default: gpu -deployment_platform: - type: str - help: "The accelerator to use for training" - choices: - - local - - huggingface - - skypilot - default: local +deploy_locally: + type: bool + help: "Whether to deploy locally" + default: True +deploy_to_huggingface: + type: bool + help: "Whether to deploy to HuggingFace Hub" + default: False +deploy_to_skypilot: + type: bool + help: "Whether to deploy to SkyPilot" + default: False dataset: type: str help: "The name of the dataset to use from HuggingFace Datasets" @@ -96,6 +100,10 @@ cloud_of_choice: - aws - gcp default: aws +metric_compare_promotion: + type: bool + help: "Whether to promote model versions based on metrics? Otherwise, latest trained will get promoted." + default: True notify_on_failures: type: bool help: "Whether to notify on pipeline failures?" diff --git a/template/gradio/app.py b/template/gradio/app.py index be81228..da7daa4 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -22,10 +22,6 @@ from os.path import dirname import gradio as gr -from zenml.logger import get_logger - -# Initialize logger -logger = get_logger(__name__) @click.command() @click.option("--tokenizer_name_or_path", default="tokenizer", help="Name or the path of the tokenizer.") @@ -55,7 +51,7 @@ def sentiment_analysis( tokenizer_name_or_path, model_name_or_path, labels, title, description, interpretation, examples ): labels = labels.split(",") - examples = [examples.split(",")] + examples = [examples] def preprocess(text): new_text = [] @@ -70,10 +66,10 @@ def softmax(x): return e_x / e_x.sum(axis=0) def analyze_text(text): - model_path = f"{dirname(__file__)}/{tokenizer_name_or_path}/" - logger.info(f"Loading model from {model_path}") - tokenizer_path = f"{dirname(__file__)}/{model_name_or_path}/" - logger.info(f"Loading tokenizer from {tokenizer_path}") + model_path = f"{dirname(__file__)}/{model_name_or_path}/" + print(f"Loading model from {model_path}") + tokenizer_path = f"{dirname(__file__)}/{tokenizer_name_or_path}/" + print(f"Loading tokenizer from {tokenizer_path}") tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) model = AutoModelForSequenceClassification.from_pretrained(model_path) diff --git a/template/gradio/serve.yaml b/template/gradio/serve.yaml index d88b480..41bc1e3 100644 --- a/template/gradio/serve.yaml +++ b/template/gradio/serve.yaml @@ -1,6 +1,16 @@ # Task name (optional), used for display purposes. name: nlp_use_case +resources: + cloud: aws # The cloud to use (optional). + + # The region to use (optional). Auto-failover will be disabled + # if this is specified. + region: us-east-1 + + # The instance type to use (optional). + instance_type: t3.large + # Working directory (optional), synced to ~/sky_workdir on the remote cluster # each time launch or exec is run with the yaml file. # @@ -16,13 +26,5 @@ setup: | echo "Setup complete." run: | - conda activate vllm - echo 'Starting vllm api server...' - python -u -m app.py \ - ----tokenizer_name $MODEL_NAME \ - --tensor-parallel-size $SKYPILOT_NUM_GPUS_PER_NODE \ - --tokenizer hf-internal-testing/llama-tokenizer 2>&1 | tee api_server.log & - echo 'Waiting for vllm api server to start...' - while ! `cat api_server.log | grep -q 'Uvicorn running on'`; do sleep 1; done - echo 'Starting gradio server...' - python vllm/examples/gradio_webserver.py \ No newline at end of file + echo 'Starting gradio app...' + python -u -m app.py \ No newline at end of file diff --git a/template/pipelines/__init__.py b/template/pipelines/__init__.py index f2e66ac..4dbc391 100644 --- a/template/pipelines/__init__.py +++ b/template/pipelines/__init__.py @@ -3,4 +3,4 @@ from .training import {{product_name}}_training_pipeline from .promoting import {{product_name}}_promote_pipeline -from .deploying import {{product_name}}_{{deployment_platform}}_deploy_pipeline +from .deploying import {{product_name}}_deploy_pipeline diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index c5d6ef5..c0bbc07 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -6,13 +6,13 @@ notify_on_failure, notify_on_success, save_model_to_deploy, -{% if deployment_platform == "local" %} +{% if deploy_locally %} deploy_locally, {% endif %} -{% if deployment_platform == "huggingface" %} +{% if deploy_to_huggingface %} deploy_to_huggingface, {% endif %} -{% if deployment_platform == "skypilot" %} +{% if deploy_to_skypilot == "skypilot" %} deploy_to_skypilot, {%- endif %} ) @@ -35,7 +35,7 @@ @pipeline( on_failure=notify_on_failure, ) -def {{product_name}}_{{deployment_platform}}_deploy_pipeline( +def {{product_name}}_deploy_pipeline( labels: Optional[List[str]] = ["Negative", "Positive"], title: Optional[str] = None, description: Optional[str] = None, @@ -43,6 +43,7 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( tokenizer_name_or_path: Optional[str] = "tokenizer", interpretation: Optional[str] = None, example: Optional[str] = None, + repo_name: Optional[str] = "{{product_name}}", ): """ Model deployment pipeline. @@ -51,12 +52,14 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( # Link all the steps together by calling them and passing the output # of one step as the input of the next step. pipeline_extra = get_pipeline_context().extra - ########## Promotion stage ########## + ########## Save Model locally stage ########## save_model_to_deploy( mlflow_model_name=pipeline_extra["mlflow_model_name"], stage=pipeline_extra["target_env"], ) -{%- if deployment_platform == "local" %} + + ########## Deployment stage ########## +{%- if deploy_locally %} deploy_locally( labels=labels, title=title, @@ -67,30 +70,32 @@ def {{product_name}}_{{deployment_platform}}_deploy_pipeline( tokenizer_name_or_path=tokenizer_name_or_path, after=["save_model_to_deploy"], ) - last_step_name = "deploy_locally" {%- endif %} -{%- if deployment_platform == "huggingface" %} + +{%- if deploy_to_huggingface %} + deploy_to_huggingface( - repo_name="{{project_name}}", - labels=labels, - title=title, - description=description, + repo_name=repo_name, after=["save_model_to_deploy"], ) - last_step_name = "deploy_to_huggingface" {%- endif %} -{%- if deployment_platform == "skypilot" %} + +{%- if deploy_to_skypilot %} + deploy_to_skypilot( - labels=labels, - title=title, - description=description, - interpretation=interpretation, - example=example, - model_name_or_path=model_name_or_path, - tokenizer_name_or_path=tokenizer_name_or_path, after=["save_model_to_deploy"], ) +{%- endif %} + +{%- if deploy_to_skypilot %} + last_step_name = "deploy_to_skypilot" +{%- elif deploy_to_huggingface %} + + last_step_name = "deploy_to_huggingface" +{%- elif deploy_locally %} + + last_step_name = "deploy_locally" {%- endif %} notify_on_success(after=[last_step_name]) diff --git a/template/run.py b/template/run.py index 2eb87ae..57e51b0 100644 --- a/template/run.py +++ b/template/run.py @@ -7,7 +7,7 @@ from pipelines import ( {{product_name}}_training_pipeline, {{product_name}}_promote_pipeline, - {{product_name}}_{{deployment_platform}}_deploy_pipeline, + {{product_name}}_deploy_pipeline, ) from zenml.logger import get_logger @@ -93,7 +93,7 @@ "--deploying-pipeline", is_flag=True, default=True, - help="Whether to run the pipeline that deploys the model to {{deployment_platform}}.", + help="Whether to run the pipeline that deploys the model to selected deployment platform.", ) @click.option( "--depployment-app-title", @@ -182,6 +182,7 @@ def main( logger.info("Promoting pipeline finished successfully!") if deploying_pipeline: + pipeline_args["enable_cache"] = False run_args_deploying = { "title": depployment_app_title, "description": depployment_app_description, @@ -190,8 +191,8 @@ def main( } pipeline_args[ "run_name" - ] = f"{{product_name}}_{{deployment_platform}}_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" - {{product_name}}_{{deployment_platform}}_deploy_pipeline.with_options(**pipeline_args)(**run_args_deploying) + ] = f"{{product_name}}_deploy_pipeline_run_{dt.now().strftime('%Y_%m_%d_%H_%M_%S')}" + {{product_name}}_deploy_pipeline.with_options(**pipeline_args)(**run_args_deploying) logger.info("Deploying pipeline finished successfully!") diff --git a/template/steps/__init__.py b/template/steps/__init__.py index e333285..f10e21c 100644 --- a/template/steps/__init__.py +++ b/template/steps/__init__.py @@ -25,13 +25,13 @@ from .deploying import ( save_model_to_deploy, -{% if deployment_platform == "local" %} +{% if deploy_locally %} deploy_locally, {% endif %} -{% if deployment_platform == "huggingface" %} +{% if deploy_to_huggingface %} deploy_to_huggingface, {% endif %} -{% if deployment_platform == "skypilot" %} +{% if deploy_to_skypilot == "skypilot" %} deploy_to_skypilot, {%- endif %} ) diff --git a/template/steps/deploying/__init__.py b/template/steps/deploying/__init__.py index a6243ce..d9a2646 100644 --- a/template/steps/deploying/__init__.py +++ b/template/steps/deploying/__init__.py @@ -2,12 +2,12 @@ from .save_model import save_model_to_deploy -{% if deployment_platform == "local" %} +{% if deploy_locally %} from .local_deployment import deploy_locally {% endif %} -{% if deployment_platform == "huggingface" %} +{% if deploy_to_huggingface %} from .huggingface_deployment import deploy_to_huggingface {% endif %} -{% if deployment_platform == "skypilot" %} +{% if deploy_to_skypilot %} from .skypilot_deployment import deploy_to_skypilot {%- endif %} \ No newline at end of file diff --git "a/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" b/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} similarity index 71% rename from "template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" rename to template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} index 0d48f37..e07adf3 100644 --- "a/template/steps/deploying/{% if deployment_platform == \"local\" %}local_deployment.py{% endif %}" +++ b/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} @@ -12,13 +12,13 @@ from zenml.logger import get_logger logger = get_logger(__name__) -@step() +@step(enable_cache=False) def deploy_locally( model_name_or_path: str, tokenizer_name_or_path: str, labels: List[str], - title: Optional[str] = "ZenML", - description: Optional[str] = "ZenML NLP Use-Case", + title: Optional[str] = "ZenML NLP Use-Case", + description: Optional[str] = "This is a demo of the ZenML NLP use-case using Gradio.", interpretation: Optional[str] = "default", example: Optional[str] = "This use-case is awesome!", ): @@ -52,8 +52,16 @@ def deploy_locally( lables = ",".join(labels) # Construct the path to the app.py file - app_path = str(os.path.join(Client().root, "gradio", "app.py")) - command = ["python", app_path, "--tokenizer_name_or_path", tokenizer_path, "--model_name_or_path", model_path, + zenml_repo_root = Client().root + if not zenml_repo_root: + logger.warning( + "You're running the `deploy_to_huggingface` step outside of a ZenML repo." + "Since the deployment step to huggingface is all about pushing the repo to huggingface, " + "this step will not work outside of a ZenML repo where the gradio folder is present." + ) + raise + app_path = str(os.path.join(zenml_repo_root, "gradio", "app.py")) + command = ["python", app_path, "--tokenizer_name_or_path", tokenizer_name_or_path, "--model_name_or_path", model_name_or_path, "--labels", lables, "--title", title, "--description", description, "--interpretation", interpretation, "--examples", example] logger.info(f"Command: {command}") diff --git "a/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}skypilot_deployment.py{% endif %}" "b/template/steps/deploying/{% if deploy_platform == \"skypilot\" %}skypilot_deployment.py{% endif %}" deleted file mode 100644 index e69de29..0000000 diff --git "a/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} similarity index 53% rename from "template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" rename to template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} index eff5f20..3e4cee6 100644 --- "a/template/steps/deploying/{% if deploy_platform == \"huggingface\" %}huggingface_deployment.py{% endif %}" +++ b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} @@ -14,9 +14,6 @@ logger = get_logger(__name__) @step() def deploy_to_huggingface( repo_name: str, - labels: Optional[List[str]] = ["Negative", "Positive"], - title: str = "ZenML", - description: str = "ZenML NLP Use-Case", ): """ @@ -28,14 +25,23 @@ def deploy_to_huggingface( The trained model and tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - lables = ",".join(labels) secret = Client().get_secret("huggingface_creds") + assert secret, "No secret found with name 'huggingface_creds'. Please create one with your `username` and `token`." huggingface_username = secret.secret_values["username"] token = secret.secret_values["token"] api = HfApi(token=token) - api.create_repo(repo_id=repo_name, repo_type="space", exist_ok=True) + hf_repo = api.create_repo(repo_id=repo_name, repo_type="space", space_sdk="gradio", exist_ok=True) + zenml_repo_root = Client().root + if not zenml_repo_root: + logger.warning( + "You're running the `deploy_to_huggingface` step outside of a ZenML repo." + "Since the deployment step to huggingface is all about pushing the repo to huggingface, " + "this step will not work outside of a ZenML repo where the gradio folder is present." + ) + raise + gradio_folder_path = os.path.join(zenml_repo_root, "gradio") space = api.upload_folder( - path="./gradio", repo_id=repo_name, repo_type="space" + folder_path=gradio_folder_path, repo_id=hf_repo.repo_id, repo_type="space", ) logger.info(f"Space created: {space}") ### YOUR CODE ENDS HERE ### diff --git a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} new file mode 100644 index 0000000..2296faf --- /dev/null +++ b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} @@ -0,0 +1,40 @@ +# {% include 'template/license_header' %} + +from typing import Optional, List + +import os +import sky +from huggingface_hub import create_branch, login, HfApi + +from zenml import step +from zenml.client import Client +from zenml.logger import get_logger + +# Initialize logger +logger = get_logger(__name__) + + +@step() +def deploy_to_skypilot( +): + """ + Args: + mlfow_model_name: The name of the model in MLFlow. + stage: The stage of the model in MLFlow. + + Returns: + The trained model and tokenizer. + """ + ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### + zenml_repo_root = Client().root + if not zenml_repo_root: + logger.warning( + "You're running the `deploy_to_huggingface` step outside of a ZenML repo." + "Since the deployment step to huggingface is all about pushing the repo to huggingface, " + "this step will not work outside of a ZenML repo where the gradio folder is present." + ) + raise + gradio_task_yaml = os.path.join(zenml_repo_root, "gradio", "serve.yaml") + task = sky.Task.from_yaml(gradio_task_yaml) + sky.launch(task, cluster_name='{{product_name}}_cluster') + ### YOUR CODE ENDS HERE ### From 502166d967ef10320e73cd2eb85ee74c1b463514 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Fri, 27 Oct 2023 04:03:16 +0100 Subject: [PATCH 12/18] deployment is done --- template/gradio/serve.yaml | 2 +- template/pipelines/deploying.py | 2 +- template/pipelines/promoting.py | 6 ++++-- template/steps/__init__.py | 2 +- ...uggingface %}huggingface_deployment.py{% endif %} | 1 + ...y_to_skypilot %}skypilot_deployment.py{% endif %} | 4 +++- ...pare_promotion %}promote_get_metric.py{% endif %} | 12 ++++++++---- ...n %}promote_metric_compare_promoter.py{% endif %} | 2 +- 8 files changed, 20 insertions(+), 11 deletions(-) diff --git a/template/gradio/serve.yaml b/template/gradio/serve.yaml index 41bc1e3..4fd76a7 100644 --- a/template/gradio/serve.yaml +++ b/template/gradio/serve.yaml @@ -27,4 +27,4 @@ setup: | run: | echo 'Starting gradio app...' - python -u -m app.py \ No newline at end of file + python app.py \ No newline at end of file diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index c0bbc07..e28f2bd 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -12,7 +12,7 @@ {% if deploy_to_huggingface %} deploy_to_huggingface, {% endif %} -{% if deploy_to_skypilot == "skypilot" %} +{% if deploy_to_skypilot %} deploy_to_skypilot, {%- endif %} ) diff --git a/template/pipelines/promoting.py b/template/pipelines/promoting.py index 75b34b9..7592be9 100644 --- a/template/pipelines/promoting.py +++ b/template/pipelines/promoting.py @@ -34,11 +34,13 @@ def {{product_name}}_promote_pipeline(): latest_version, current_version = promote_get_versions() {%- if metric_compare_promotion %} latest_metric = promote_get_metric( - metric="accuracy", + name=pipeline_extra["mlflow_model_name"], + metric="eval_loss", version=latest_version, ) current_metric = promote_get_metric( - metric="accuracy", + name=pipeline_extra["mlflow_model_name"], + metric="eval_loss", version=current_version, ) diff --git a/template/steps/__init__.py b/template/steps/__init__.py index f10e21c..f1ba5a5 100644 --- a/template/steps/__init__.py +++ b/template/steps/__init__.py @@ -31,7 +31,7 @@ {% if deploy_to_huggingface %} deploy_to_huggingface, {% endif %} -{% if deploy_to_skypilot == "skypilot" %} +{% if deploy_to_skypilot %} deploy_to_skypilot, {%- endif %} ) diff --git a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} index 3e4cee6..8cec74f 100644 --- a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} @@ -1,5 +1,6 @@ # {% include 'template/license_header' %} +import os from typing import Optional, List from huggingface_hub import create_branch, login, HfApi diff --git a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} index 2296faf..ed63650 100644 --- a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} @@ -3,6 +3,7 @@ from typing import Optional, List import os +import re import sky from huggingface_hub import create_branch, login, HfApi @@ -36,5 +37,6 @@ def deploy_to_skypilot( raise gradio_task_yaml = os.path.join(zenml_repo_root, "gradio", "serve.yaml") task = sky.Task.from_yaml(gradio_task_yaml) - sky.launch(task, cluster_name='{{product_name}}_cluster') + cluster_name = re.sub(r'[^a-zA-Z0-9]+', '-', '{{product_name}}-cluster') + sky.launch(task, cluster_name=cluster_name) ### YOUR CODE ENDS HERE ### diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} index 297eddf..c324ea0 100644 --- a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} @@ -2,7 +2,7 @@ from typing_extensions import Annotated - +import mlflow from zenml import get_step_context, step from zenml.client import Client from zenml.logger import get_logger @@ -14,6 +14,7 @@ model_registry = Client().active_stack.model_registry @step def promote_get_metric( + name: str, metric: str, version: str, ) -> Annotated[float, "metric"]: @@ -39,10 +40,13 @@ def promote_get_metric( """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - model_version = model_registry.get_model_version(version) - mlflow_run = mlflow.get_run(run_id=model_version.metadata["mlflow_run_id"]) + model_version = model_registry.get_model_version( + name = name, + version = version + ) + mlflow_run = mlflow.get_run(run_id=model_version.metadata.mlflow_run_id) logger.info("Getting metric from MLFlow run %s", mlflow_run.info.run_id) - metric = mlflow_run.data.metrics[metric] + metric = mlflow_run.data.metrics.get(metric) ### YOUR CODE ENDS HERE ### return metric diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} index d055885..dc5ddb9 100644 --- a/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} @@ -55,7 +55,7 @@ def promote_metric_compare_promoter( f"Latest model metric={latest_metric:.6f}\n" f"Current model metric={current_metric:.6f}" ) - if latest_metric > current_metric: + if latest_metric <= current_metric: logger.info( "Latest model versions outperformed current versions - promoting latest" ) From 3d9e4345768da1fa6bebeeaa37a0bc728e63f7eb Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:15:11 +0100 Subject: [PATCH 13/18] adding docstrings --- template/gradio/app.py | 38 ++++++++++++++++--- template/gradio/serve.yaml | 5 ++- template/pipelines/deploying.py | 30 +++++++++++---- template/pipelines/promoting.py | 6 +++ template/pipelines/training.py | 31 ++++++++------- template/run.py | 2 +- template/steps/dataset_loader/__init__.py | 1 - template/steps/dataset_loader/data_loader.py | 10 +---- template/steps/deploying/save_model.py | 10 +++-- ...y_locally %}local_deployment.py{% endif %} | 19 +++++++--- ...ace %}huggingface_deployment.py{% endif %} | 9 ++--- ...ypilot %}skypilot_deployment.py{% endif %} | 10 ++--- ...omotion %}promote_get_metric.py{% endif %} | 23 ++++------- .../steps/registrer/model_log_register.py | 3 +- template/steps/tokenzation/tokenization.py | 31 +++++---------- template/steps/training/model_trainer.py | 14 +++---- 16 files changed, 137 insertions(+), 105 deletions(-) diff --git a/template/gradio/app.py b/template/gradio/app.py index da7daa4..06d5871 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -13,11 +13,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# - +from typing import Optional import click import numpy as np -import os from transformers import AutoModelForSequenceClassification, AutoTokenizer from os.path import dirname @@ -34,7 +32,7 @@ ) @click.option( "--description", - default="Sentiment Analyzer", + default="Text Classification - Sentiment Analysis - ZenML - Gradio", help="Description of the Gradio interface.", ) @click.option( @@ -48,12 +46,40 @@ help="Comma-separated list of examples to show in the Gradio interface.", ) def sentiment_analysis( - tokenizer_name_or_path, model_name_or_path, labels, title, description, interpretation, examples + tokenizer_name_or_path: Optional[str], + model_name_or_path: Optional[str], + labels: Optional[str], + title: Optional[str], + description: Optional[str], + interpretation: Optional[str], + examples: Optional[str], ): + """Launches a Gradio interface for sentiment analysis. + + This function launches a Gradio interface for text-classification. + It loads a model and a tokenizer from the provided paths and uses + them to predict the sentiment of the input text. + + Args: + tokenizer_name_or_path (str): Name or the path of the tokenizer. + model_name_or_path (str): Name or the path of the model. + labels (str): Comma-separated list of labels. + title (str): Title of the Gradio interface. + description (str): Description of the Gradio interface. + interpretation (str): Interpretation mode for the Gradio interface. + examples (str): Comma-separated list of examples to show in the Gradio interface. + """ labels = labels.split(",") examples = [examples] + def preprocess(text: str) -> str: + """Preprocesses the text. - def preprocess(text): + Args: + text (str): Input text. + + Returns: + str: Preprocessed text. + """ new_text = [] for t in text.split(" "): t = "@user" if t.startswith("@") and len(t) > 1 else t diff --git a/template/gradio/serve.yaml b/template/gradio/serve.yaml index 4fd76a7..3ef8a86 100644 --- a/template/gradio/serve.yaml +++ b/template/gradio/serve.yaml @@ -1,7 +1,8 @@ # Task name (optional), used for display purposes. -name: nlp_use_case +name: {{project_name}}} resources: +{%- if cloud_of_choice == 'aws' %} cloud: aws # The cloud to use (optional). # The region to use (optional). Auto-failover will be disabled @@ -10,6 +11,8 @@ resources: # The instance type to use (optional). instance_type: t3.large +{%- else %} + cloud: gcp # The cloud to use (optional). # Working directory (optional), synced to ~/sky_workdir on the remote cluster # each time launch or exec is run with the yaml file. diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index e28f2bd..2a920c9 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -26,7 +26,7 @@ orchestrator = Client().active_stack.orchestrator # Check if orchestrator flavor is either default or skypilot -if orchestrator.flavor not in ["local", "vm_aws", "vm_gcp"]: +if orchestrator.flavor not in ["local"]: raise RuntimeError( "Your active stack needs to contain a default or skypilot orchestrator for " "the deployment pipeline to work." @@ -47,19 +47,33 @@ def {{product_name}}_deploy_pipeline( ): """ Model deployment pipeline. + + This pipelines deploys latest model on mlflow registry that matches + the given stage, to one of the supported deployment targets. + + Args: + labels: List of labels for the model. + title: Title for the model. + description: Description for the model. + model_name_or_path: Name or path of the model. + tokenizer_name_or_path: Name or path of the tokenizer. + interpretation: Interpretation for the model. + example: Example for the model. + repo_name: Name of the repository to deploy to HuggingFace Hub. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### # Link all the steps together by calling them and passing the output # of one step as the input of the next step. pipeline_extra = get_pipeline_context().extra - ########## Save Model locally stage ########## + + ########## Save Model locally ########## save_model_to_deploy( mlflow_model_name=pipeline_extra["mlflow_model_name"], stage=pipeline_extra["target_env"], ) - ########## Deployment stage ########## {%- if deploy_locally %} + ########## Deploy Locally ########## deploy_locally( labels=labels, title=title, @@ -70,31 +84,31 @@ def {{product_name}}_deploy_pipeline( tokenizer_name_or_path=tokenizer_name_or_path, after=["save_model_to_deploy"], ) + {%- endif %} {%- if deploy_to_huggingface %} - + ########## Deploy to HuggingFace ########## deploy_to_huggingface( repo_name=repo_name, after=["save_model_to_deploy"], ) + {%- endif %} {%- if deploy_to_skypilot %} - + ########## Deploy to Skypilot ########## deploy_to_skypilot( after=["save_model_to_deploy"], ) + {%- endif %} {%- if deploy_to_skypilot %} - last_step_name = "deploy_to_skypilot" {%- elif deploy_to_huggingface %} - last_step_name = "deploy_to_huggingface" {%- elif deploy_locally %} - last_step_name = "deploy_locally" {%- endif %} diff --git a/template/pipelines/promoting.py b/template/pipelines/promoting.py index 7592be9..896871f 100644 --- a/template/pipelines/promoting.py +++ b/template/pipelines/promoting.py @@ -25,11 +25,17 @@ def {{product_name}}_promote_pipeline(): """ Model promotion pipeline. + + This is a pipeline that promotes the best model to the chosen + stage, e.g. Production or Staging. Based on a metric comparison + between the latest and the currently promoted model version, + or just the latest model version. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### # Link all the steps together by calling them and passing the output # of one step as the input of the next step. pipeline_extra = get_pipeline_context().extra + ########## Promotion stage ########## latest_version, current_version = promote_get_versions() {%- if metric_compare_promotion %} diff --git a/template/pipelines/training.py b/template/pipelines/training.py index ab6fb16..8ef50a3 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -41,30 +41,33 @@ def {{product_name}}_training_pipeline( """ Model training pipeline. - This is a pipeline that loads the data, processes it and splits - it into train and test sets, then search for best hyperparameters, - trains and evaluates a model. + This is a pipeline that loads the datataset and tokenzier, + tokenizes the dataset, trains a model and registers the model + to the model registry. Args: - test_size: Size of holdout set for training 0.0..1.0 - drop_na: If `True` NA values will be removed from dataset - normalize: If `True` dataset will be normalized with MinMaxScaler - drop_columns: List of columns to drop from dataset - random_seed: Seed of random generator, - min_train_accuracy: Threshold to stop execution if train set accuracy is lower - min_test_accuracy: Threshold to stop execution if test set accuracy is lower - fail_on_accuracy_quality_gates: If `True` and `min_train_accuracy` or `min_test_accuracy` - are not met - execution will be interrupted early - + lower_case: Whether to convert all text to lower case. + padding: Padding strategy. + max_seq_length: Maximum sequence length. + text_column: Name of the text column. + label_column: Name of the label column. + train_batch_size: Training batch size. + eval_batch_size: Evaluation batch size. + num_epochs: Number of epochs. + learning_rate: Learning rate. + weight_decay: Weight decay. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### # Link all the steps together by calling them and passing the output # of one step as the input of the next step. pipeline_extra = get_pipeline_context().extra - ########## Tokenization stage ########## + + ########## Load Dataset stage ########## dataset = data_loader( shuffle=True, ) + + ########## Tokenization stage ########## tokenizer = tokenizer_loader( lower_case=lower_case ) diff --git a/template/run.py b/template/run.py index 57e51b0..bb7d76b 100644 --- a/template/run.py +++ b/template/run.py @@ -92,7 +92,7 @@ @click.option( "--deploying-pipeline", is_flag=True, - default=True, + default=False, help="Whether to run the pipeline that deploys the model to selected deployment platform.", ) @click.option( diff --git a/template/steps/dataset_loader/__init__.py b/template/steps/dataset_loader/__init__.py index e51200a..1474d91 100644 --- a/template/steps/dataset_loader/__init__.py +++ b/template/steps/dataset_loader/__init__.py @@ -1,4 +1,3 @@ # {% include 'template/license_header' %} - from .data_loader import data_loader diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index 413e803..1e6c22c 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -14,14 +14,8 @@ def data_loader( """ Data loader step. - This step reads data from an external source like a file, - database or 3rd party library, then formats it and returns it as a step - output artifact. - - This step is parameterized using the `HFSentimentAnalysisDataset` class, which - allows you to configure the step independently of the step code, before - running it in a pipeline. In this example, the step can be configured to - load different built-in scikit-learn datasets. + This step reads data from a Huggingface dataset or a CSV files and returns + a Huggingface dataset. Data loader steps should have caching disabled if they are not deterministic (i.e. if they data they load from the external source can be different when diff --git a/template/steps/deploying/save_model.py b/template/steps/deploying/save_model.py index cd273d7..bf94b4c 100644 --- a/template/steps/deploying/save_model.py +++ b/template/steps/deploying/save_model.py @@ -21,12 +21,16 @@ def save_model_to_deploy( stage: str, ): """ + This step saves the latest model and tokenizer to the local filesystem. + + Note: It's recommended to use this step in a pipeline that is run locally, + using the `local` orchestrator flavor because this step saves the model + and tokenizer to the local filesystem, that will later then be used by the deployment + steps. + Args: mlfow_model_name: The name of the model in MLFlow. stage: The stage of the model in MLFlow. - - Returns: - The trained model and tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### logger.info(f" Loading latest version of model {mlflow_model_name} for stage {stage}...") diff --git a/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} index e07adf3..ca07c79 100644 --- a/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} @@ -21,15 +21,22 @@ def deploy_locally( description: Optional[str] = "This is a demo of the ZenML NLP use-case using Gradio.", interpretation: Optional[str] = "default", example: Optional[str] = "This use-case is awesome!", -): +) -> Annotated[str, "process_id"]: """ - + This step deploy the model locally by starting a Gradio app + as a separate process in the background. + Args: - mlfow_model_name: The name of the model in MLFlow. - stage: The stage of the model in MLFlow. + model_name_or_path: The name or path of the model to use. + tokenizer_name_or_path: The name or path of the tokenizer to use. + labels: The labels of the model. + title: The title of the Gradio app. + description: The description of the Gradio app. + interpretation: The interpretation of the Gradio app. + example: The example of the Gradio app. Returns: - The trained model and tokenizer. + The process ID of the Gradio app. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### def start_gradio_app(command: list[str]) -> int: @@ -70,3 +77,5 @@ def deploy_locally( logger.info(f"Process ID: {pid}") logger.info(f"To kill the process, run: kill -9 {pid}") ### YOUR CODE ENDS HERE ### + + return pid \ No newline at end of file diff --git a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} index 8cec74f..fa69e36 100644 --- a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} @@ -17,13 +17,10 @@ def deploy_to_huggingface( repo_name: str, ): """ - - Args: - mlfow_model_name: The name of the model in MLFlow. - stage: The stage of the model in MLFlow. + This step deploy the model to huggingface. - Returns: - The trained model and tokenizer. + Args: + repo_name: The name of the repo to create/use on huggingface. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### secret = Client().get_secret("huggingface_creds") diff --git a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} index ed63650..5106b63 100644 --- a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} @@ -19,12 +19,10 @@ logger = get_logger(__name__) def deploy_to_skypilot( ): """ - Args: - mlfow_model_name: The name of the model in MLFlow. - stage: The stage of the model in MLFlow. + This step deploy the model to a VM using SkyPilot. - Returns: - The trained model and tokenizer. + This step requires `skypilot` to be installed. + aswell as a configured cloud account locally (e.g. AWS, GCP, Azure). """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### zenml_repo_root = Client().root @@ -38,5 +36,5 @@ def deploy_to_skypilot( gradio_task_yaml = os.path.join(zenml_repo_root, "gradio", "serve.yaml") task = sky.Task.from_yaml(gradio_task_yaml) cluster_name = re.sub(r'[^a-zA-Z0-9]+', '-', '{{product_name}}-cluster') - sky.launch(task, cluster_name=cluster_name) + sky.launch(task, cluster_name=cluster_name, detach_setup=True) ### YOUR CODE ENDS HERE ### diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} index c324ea0..b444ed7 100644 --- a/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_get_metric.py{% endif %} @@ -6,7 +6,6 @@ import mlflow from zenml import get_step_context, step from zenml.client import Client from zenml.logger import get_logger -from zenml.integrations.mlflow.experiment_trackers import MLFlowExperimentTracker logger = get_logger(__name__) @@ -18,25 +17,19 @@ def promote_get_metric( metric: str, version: str, ) -> Annotated[float, "metric"]: - """Get metric for comparison for one model deployment. + """Get metric for comparison for promoting a model. - This is an example of a metric calculation step. It get a model deployment - service and computes metric on recent test dataset. - - This step is parameterized, which allows you to configure the step - independently of the step code, before running it in a pipeline. - In this example, the step can be configured to use different input data. - See the documentation for more information: - - https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines + This is an example of a metric retrieval step. It is used to retrieve + a metric from an MLFlow run, that is linked to a model version in the + model registry. This step is used in the `promote_model` pipeline. Args: - dataset_tst: The test dataset. - deployment_service: Model version deployment. + name: Name of the model registered in the model registry. + metric: Name of the metric to be retrieved. + version: Version of the model to be retrieved. Returns: - Metric value for a given deployment on test set. - + Metric value for a given model version. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### diff --git a/template/steps/registrer/model_log_register.py b/template/steps/registrer/model_log_register.py index 97c150d..5ccc0c0 100644 --- a/template/steps/registrer/model_log_register.py +++ b/template/steps/registrer/model_log_register.py @@ -38,7 +38,7 @@ def register_model( Register model to MLFlow. This step takes in a model and tokenizer artifact previously loaded and pre-processed by - other steps in your pipeline, then registers the model to MLFlow for deployment. + other steps in your pipeline, then registers the model to MLFlow registry. Model training steps should have caching disabled if they are not deterministic (i.e. if the model training involve some random processes like initializing @@ -52,7 +52,6 @@ def register_model( The trained model and tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - # Log the model components = { "model": model, "tokenizer": tokenizer, diff --git a/template/steps/tokenzation/tokenization.py b/template/steps/tokenzation/tokenization.py index bf79485..7810e79 100644 --- a/template/steps/tokenzation/tokenization.py +++ b/template/steps/tokenzation/tokenization.py @@ -21,30 +21,16 @@ def tokenization_step( """ Tokenization step. - This step tokenizes the input dataset using a pre-trained tokenizer. It - takes in a dataset as an step input artifact and returns the tokenized - dataset as an output artifact. - - The tokenization process includes padding, truncation, and addition of - labels. The maximum sequence length for tokenization is determined based - on the dataset. - - This step is parameterized using the `DataSplitterStepParameters` class, - which allows you to configure the step independently of the step code, - before running it in a pipeline. In this example, the step can be configured - to use a different random seed, change the split ratio, or control whether - to shuffle or stratify the split. See the documentation for more - information: - - https://docs.zenml.io/user-guide/starter-guide/cache-previous-executions + This step tokenizes the dataset using the tokenizer and returns the tokenized + dataset in a Huggingface DatasetDict format. Args: - padding: Padding method for tokenization. - max_seq_length: Maximum sequence length for tokenization. - text_column: Column name for text data in the dataset. - label_column: Column name for label data in the dataset. - tokenizer: Pre-trained tokenizer for tokenization. - dataset: The dataset to tokenize. + tokenizer: The tokenizer to use for tokenization. + dataset: The dataset to be tokenized. + padding: Padding strategy. + max_seq_length: Maximum sequence length. + text_column: Name of the text column. + label_column: Name of the label column. Returns: The tokenized dataset. @@ -88,4 +74,5 @@ def preprocess_function(examples): # Set the format of the tokenized dataset tokenized_datasets.set_format("torch") ### YOUR CODE ENDS HERE ### + return tokenized_datasets diff --git a/template/steps/training/model_trainer.py b/template/steps/training/model_trainer.py index e55c5bf..d38653f 100644 --- a/template/steps/training/model_trainer.py +++ b/template/steps/training/model_trainer.py @@ -60,16 +60,16 @@ def model_trainer( Args: - hf_pretrained_model: The pre-trained model. tokenized_dataset: The tokenized dataset. tokenizer: The tokenizer. num_labels: The number of labels. - train_batch_size: The training batch size. - num_epochs: The number of epochs. - learning_rate: The learning rate. - load_best_model_at_end: Whether to load the best model at the end. - eval_batch_size: The evaluation batch size. - weight_decay: The weight decay. + train_batch_size: Training batch size. + num_epochs: Number of epochs. + learning_rate: Learning rate. + load_best_model_at_end: Whether to load the best model at the end of training. + eval_batch_size: Evaluation batch size. + weight_decay: Weight decay. + mlflow_model_name: The name of the model in MLFlow. Returns: The trained model and tokenizer. From 199eefdc026f4b0f6f2d0b1e03c1f4238eec2f7d Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Sat, 28 Oct 2023 15:14:16 +0100 Subject: [PATCH 14/18] readme --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 69 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 448b80b..052835c 100644 --- a/README.md +++ b/README.md @@ -1 +1,69 @@ -# 💫 ZenML End-to-End Natural Language Processing Project Template \ No newline at end of file +# 💫 ZenML End-to-End NLP Training and Deployment Project Template + +This project template is designed to help you get started with training and deploying NLP models using the ZenML framework. It provides a comprehensive set of steps and pipelines to cover major use cases of NLP model development, including dataset loading, tokenization, model training, model registration, and deployment. + +## 📃 Template Parameters + +| Parameter | Description | Default | +|-----------|-------------|---------| +| Name | The name of the person/entity holding the copyright | ZenML GmbH | +| Email | The email of the person/entity holding the copyright | info@zenml.io | +| Project Name | Short name for your project | ZenML NLP project | +| Project Version | The version of your project | 0.0.1 | +| Project License | The license under which your project will be released | Apache Software License 2.0 | +| Technical product name | The technical name to prefix all tech assets (pipelines, models, etc.) | nlp_use_case | +| Target environment | The target environment for deployments/promotions | staging | +| Use metric-based promotion | Whether to compare metric of interest to make model version promotion | True | +| Notifications on failure | Whether to notify about pipeline failures | True | +| Notifications on success | Whether to notify about pipeline successes | False | +| ZenML Server URL | Optional URL of a remote ZenML server for support scripts | - | + +## 🚀 Generate a ZenML Project + +To generate a project from this template, make sure you have ZenML and its `templates` extras installed: + +```bash +pip install zenml[templates] +``` + +Then, run the following command to generate the project: + +```bash +zenml init --template nlp-template +``` + +You will be prompted to provide values for the template parameters. If you want to use the default values, you can add the `--template-with-defaults` flag to the command. + +## 🧰 How this template is implemented + +This template provides a set of pipelines and steps to cover the end-to-end process of training and deploying NLP models. Here is an overview of the main components: + +### Dataset Loading + +The template includes a step for loading the dataset from the HuggingFace Datasets library. You can choose from three available datasets: financial_news, airline_reviews, and imbd_reviews. + +### Tokenization + +The tokenization step preprocesses the dataset by tokenizing the text data using the tokenizer provided by the HuggingFace Models library. You can choose from three available models: bert-base-uncased, roberta-base, and distilbert-base-cased. + +### Model Training + +The training pipeline consists of several steps, including model architecture search, hyperparameter tuning, model training, and model evaluation. The best model architecture and hyperparameters are selected based on the performance on the validation set. The trained model is then evaluated on the holdout set to assess its performance. + +### Model Registration and Promotion + +After training, the best model version is registered in the ZenML Model Registry. The template provides an option to promote the model version based on a specified metric of interest. If metric-based promotion is enabled, the template compares the metric value of the new model version with the metric value of the current production model version and promotes the new version if it performs better. + +### Batch Inference + +The template includes a batch inference pipeline that loads the inference dataset, preprocesses it using the same tokenizer as during training, and runs predictions using the deployed model version. The predictions are stored as an artifact for future use. + +### Deployment Options + +The template provides options to deploy the trained model locally or to the HuggingFace Hub. You can choose whether to deploy locally or to the HuggingFace Hub by setting the `deploy_locally` and `deploy_to_huggingface` parameters. + +## Next Steps + +Once you have generated the project using this template, you can explore the generated code and customize it to fit your specific NLP use case. The README.md file in the generated project provides further instructions on how to set up and run the project. + +Happy coding with ZenML and NLP! \ No newline at end of file From 3af60187024038bd5ac77785f47f1a3ccfaceac8 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Sat, 28 Oct 2023 16:37:31 +0100 Subject: [PATCH 15/18] update error in serve and delete config.py --- template/config.py | 83 -------------------------------------- template/gradio/serve.yaml | 2 +- 2 files changed, 1 insertion(+), 84 deletions(-) delete mode 100644 template/config.py diff --git a/template/config.py b/template/config.py deleted file mode 100644 index 60d916e..0000000 --- a/template/config.py +++ /dev/null @@ -1,83 +0,0 @@ -# {% include 'template/license_header' %} - - -from artifacts.model_metadata import ModelMetadata -from pydantic import BaseConfig - -from zenml.config import DockerSettings -from utils.misc import ( - HFSentimentAnalysisDataset, - HFPretrainedModel, - HFPretrainedTokenizer, -) -from zenml.integrations.constants import ( -{%- if cloud_of_choice == 'aws' %} - SKYPILOT_AWS, - AWS, - S3, -{%- endif %} -{%- if cloud_of_choice == 'gcp' %} - SKYPILOT_GCP, - GCP, -{%- endif %} - HUGGINGFACE, - PYTORCH, - MLFLOW, - SLACK, - -) -from zenml.model_registries.base_model_registry import ModelVersionStage - -PIPELINE_SETTINGS = dict( - docker=DockerSettings( - required_integrations=[ - {%- if cloud_of_choice == 'aws' %} - SKYPILOT_AWS, - AWS, - S3, - {%- endif %} - {%- if cloud_of_choice == 'gcp' %} - SKYPILOT_GCP, - GCP, - {%- endif %} - HUGGINGFACE, - PYTORCH, - MLFLOW, - SLACK, - ], - requirements=[ - "accelerate", - ], - ) -) - -DEFAULT_PIPELINE_EXTRAS = dict( - notify_on_success={{notify_on_successes}}, - notify_on_failure={{notify_on_failures}} -) - -class MetaConfig(BaseConfig): -{%- if dataset == 'imbd_reviews' %} - dataset = HFSentimentAnalysisDataset.imbd_reviews -{%- endif %} -{%- if dataset == 'airline_reviews' %} - dataset = HFSentimentAnalysisDataset.airline_reviews -{%- else %} - dataset = HFSentimentAnalysisDataset.financial_news -{%- endif %} -{%- if model == 'gpt2' %} - tokenizer = HFPretrainedTokenizer.gpt2 - model = HFPretrainedModel.gpt2 -{%- else %} - tokenizer = HFPretrainedTokenizer.bert - model = HFPretrainedModel.bert -{%- endif %} - pipeline_name_training = "{{product_name}}_training" - mlflow_model_name = "{{product_name}}_model" -{%- if target_environment == 'production' %} - target_env = ModelVersionStage.PRODUCTION -{%- else %} - target_env = ModelVersionStage.STAGING -{%- endif %} - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### \ No newline at end of file diff --git a/template/gradio/serve.yaml b/template/gradio/serve.yaml index 3ef8a86..62797fe 100644 --- a/template/gradio/serve.yaml +++ b/template/gradio/serve.yaml @@ -13,7 +13,7 @@ resources: instance_type: t3.large {%- else %} cloud: gcp # The cloud to use (optional). - +{%- endif %} # Working directory (optional), synced to ~/sky_workdir on the remote cluster # each time launch or exec is run with the yaml file. # From 840f89ab32e72d7a998a6a62bcba5c42321989c5 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Thu, 2 Nov 2023 13:38:11 +0100 Subject: [PATCH 16/18] Apply suggestions from code review Co-authored-by: Alex Strick van Linschoten --- README.md | 2 +- copier.yml | 4 ++-- template/pipelines/training.py | 2 +- template/run.py | 8 ++++---- template/steps/dataset_loader/data_loader.py | 2 +- template/steps/deploying/save_model.py | 2 +- ... if deploy_locally %}local_deployment.py{% endif %} | 4 ++-- ..._huggingface %}huggingface_deployment.py{% endif %} | 6 +++--- ...loy_to_skypilot %}skypilot_deployment.py{% endif %} | 4 ++-- ...ion %}promote_metric_compare_promoter.py{% endif %} | 10 +++++----- template/steps/registrer/model_log_register.py | 2 +- 11 files changed, 23 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 14dbeb0..7b6936b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ This template provides a set of pipelines and steps to cover the end-to-end proc ### Dataset Loading -The template includes a step for loading the dataset from the HuggingFace Datasets library. You can choose from three available datasets: financial_news, airline_reviews, and imbd_reviews. +The template includes a step for loading the dataset from the HuggingFace Datasets library. You can choose from three available datasets: financial_news, airline_reviews, and imdb_reviews. ### Tokenization diff --git a/copier.yml b/copier.yml index 3fc6fe3..e073e48 100644 --- a/copier.yml +++ b/copier.yml @@ -49,7 +49,7 @@ email: when: "{{ open_source_license != 'none' }}" product_name: type: str - help: The technical name of the data product you are building, Make sure it's one word and all lowercase (e.g. nlp_use_case) + help: The technical name of the data product you are building. Make sure it's one word and all lowercase (e.g. nlp_use_case) default: nlp_use_case target_environment: type: str @@ -102,7 +102,7 @@ cloud_of_choice: default: aws metric_compare_promotion: type: bool - help: "Whether to promote model versions based on metrics? Otherwise, latest trained will get promoted." + help: "Whether to promote model versions based on metrics. Otherwise, latest trained model will get promoted." default: True notify_on_failures: type: bool diff --git a/template/pipelines/training.py b/template/pipelines/training.py index 8ef50a3..331fd82 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -41,7 +41,7 @@ def {{product_name}}_training_pipeline( """ Model training pipeline. - This is a pipeline that loads the datataset and tokenzier, + This is a pipeline that loads the dataset and tokenizer, tokenizes the dataset, trains a model and registers the model to the model registry. diff --git a/template/run.py b/template/run.py index 11a719e..0d7f430 100644 --- a/template/run.py +++ b/template/run.py @@ -96,25 +96,25 @@ help="Whether to run the pipeline that deploys the model to selected deployment platform.", ) @click.option( - "--depployment-app-title", + "--deployment-app-title", default="Sentiment Analyzer", type=click.STRING, help="Title of the Gradio interface.", ) @click.option( - "--depployment-app-description", + "--deployment-app-description", default="Sentiment Analyzer", type=click.STRING, help="Description of the Gradio interface.", ) @click.option( - "--depployment-app-interpretation", + "--deployment-app-interpretation", default="default", type=click.STRING, help="Interpretation mode for the Gradio interface.", ) @click.option( - "--depployment-app-example", + "--deployment-app-example", default="", type=click.STRING, help="Comma-separated list of examples to show in the Gradio interface.", diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index 1e6c22c..0604829 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -32,7 +32,7 @@ def data_loader( {%- if dataset == 'financial_news' %} dataset = load_dataset("zeroshot/twitter-financial-news-sentiment") {%- endif %} - {%- if dataset == 'imbd_reviews' %} + {%- if dataset == 'imdb_reviews' %} dataset = load_dataset("imdb")["train"] dataset = dataset.train_test_split(test_size=0.25, shuffle=shuffle) {%- endif %} diff --git a/template/steps/deploying/save_model.py b/template/steps/deploying/save_model.py index bf94b4c..aaa4e9c 100644 --- a/template/steps/deploying/save_model.py +++ b/template/steps/deploying/save_model.py @@ -25,7 +25,7 @@ def save_model_to_deploy( Note: It's recommended to use this step in a pipeline that is run locally, using the `local` orchestrator flavor because this step saves the model - and tokenizer to the local filesystem, that will later then be used by the deployment + and tokenizer to the local filesystem that will later then be used by the deployment steps. Args: diff --git a/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} index ca07c79..235b4ea 100644 --- a/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_locally %}local_deployment.py{% endif %} @@ -23,7 +23,7 @@ def deploy_locally( example: Optional[str] = "This use-case is awesome!", ) -> Annotated[str, "process_id"]: """ - This step deploy the model locally by starting a Gradio app + This step deploys the model locally by starting a Gradio app as a separate process in the background. Args: @@ -62,7 +62,7 @@ def deploy_locally( zenml_repo_root = Client().root if not zenml_repo_root: logger.warning( - "You're running the `deploy_to_huggingface` step outside of a ZenML repo." + "You're running the `deploy_to_huggingface` step outside of a ZenML repo. " "Since the deployment step to huggingface is all about pushing the repo to huggingface, " "this step will not work outside of a ZenML repo where the gradio folder is present." ) diff --git a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} index fa69e36..d2df257 100644 --- a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} @@ -24,15 +24,15 @@ def deploy_to_huggingface( """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### secret = Client().get_secret("huggingface_creds") - assert secret, "No secret found with name 'huggingface_creds'. Please create one with your `username` and `token`." + assert secret, "No secret found with name 'huggingface_creds'. Please create one that includes your `username` and `token`." huggingface_username = secret.secret_values["username"] - token = secret.secret_values["token"] + huggingface_token = secret.secret_values["token"] api = HfApi(token=token) hf_repo = api.create_repo(repo_id=repo_name, repo_type="space", space_sdk="gradio", exist_ok=True) zenml_repo_root = Client().root if not zenml_repo_root: logger.warning( - "You're running the `deploy_to_huggingface` step outside of a ZenML repo." + "You're running the `deploy_to_huggingface` step outside of a ZenML repo. " "Since the deployment step to huggingface is all about pushing the repo to huggingface, " "this step will not work outside of a ZenML repo where the gradio folder is present." ) diff --git a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} index 5106b63..bd6d2e0 100644 --- a/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_skypilot %}skypilot_deployment.py{% endif %} @@ -28,9 +28,9 @@ def deploy_to_skypilot( zenml_repo_root = Client().root if not zenml_repo_root: logger.warning( - "You're running the `deploy_to_huggingface` step outside of a ZenML repo." + "You're running the `deploy_to_huggingface` step outside of a ZenML repo. " "Since the deployment step to huggingface is all about pushing the repo to huggingface, " - "this step will not work outside of a ZenML repo where the gradio folder is present." + "this step will only work within a ZenML repo where the gradio folder is present." ) raise gradio_task_yaml = os.path.join(zenml_repo_root, "gradio", "serve.yaml") diff --git a/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} b/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} index dc5ddb9..0152776 100644 --- a/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} +++ b/template/steps/promotion/{% if metric_compare_promotion %}promote_metric_compare_promoter.py{% endif %} @@ -21,13 +21,13 @@ def promote_metric_compare_promoter( """Try to promote trained model. This is an example of a model promotion step. It gets precomputed - metrics for 2 model version: latest and currently promoted to target environment - (Production, Staging, etc) and compare than in order to define + metrics for two model versions, the latest and currently promoted to target environment + (Production, Staging, etc) and compare them in order to check if newly trained model is performing better or not. If new model - version is better by metric - it will get relevant + version is better as per the metric - it will get relevant tag, otherwise previously promoted model version will remain. - If the latest version is the only one - it will get promoted automatically. + If the latest version is the only one, it will get promoted automatically. This step is parameterized, which allows you to configure the step independently of the step code, before running it in a pipeline. @@ -40,7 +40,7 @@ def promote_metric_compare_promoter( latest_metric: Recently trained model metric results. current_metric: Previously promoted model metric results. latest_version: Recently trained model version. - current_version:Previously promoted model version. + current_version: Previously promoted model version. """ diff --git a/template/steps/registrer/model_log_register.py b/template/steps/registrer/model_log_register.py index 5ccc0c0..3934f4d 100644 --- a/template/steps/registrer/model_log_register.py +++ b/template/steps/registrer/model_log_register.py @@ -23,7 +23,7 @@ experiment_tracker, MLFlowExperimentTracker ): raise RuntimeError( - "Your active stack needs to contain a MLFlow experiment tracker for " + "Your active stack needs to contain an MLFlow experiment tracker for " "this example to work." ) From 23eeb4daa6e5cc85fcf1c8153f1f6da8c03a1ee4 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Tue, 31 Oct 2023 14:07:46 +0100 Subject: [PATCH 17/18] remove artifacts folder --- template/artifacts/__init__.py | 1 - template/artifacts/materializer.py | 89 ---------------------------- template/artifacts/model_metadata.py | 62 ------------------- 3 files changed, 152 deletions(-) delete mode 100644 template/artifacts/__init__.py delete mode 100644 template/artifacts/materializer.py delete mode 100644 template/artifacts/model_metadata.py diff --git a/template/artifacts/__init__.py b/template/artifacts/__init__.py deleted file mode 100644 index 4bc11e5..0000000 --- a/template/artifacts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# {% include 'template/license_header' %} diff --git a/template/artifacts/materializer.py b/template/artifacts/materializer.py deleted file mode 100644 index 40c89a3..0000000 --- a/template/artifacts/materializer.py +++ /dev/null @@ -1,89 +0,0 @@ -# {% include 'template/license_header' %} - - -import json -import os -from typing import Type - -from zenml.enums import ArtifactType -from zenml.io import fileio -from zenml.materializers.base_materializer import BaseMaterializer - -from artifacts.model_metadata import ModelMetadata - - -class ModelMetadataMaterializer(BaseMaterializer): - ASSOCIATED_TYPES = (ModelMetadata,) - ASSOCIATED_ARTIFACT_TYPE = ArtifactType.STATISTICS - - def load(self, data_type: Type[ModelMetadata]) -> ModelMetadata: - """Read from artifact store. - - Args: - data_type: What type the artifact data should be loaded as. - - Raises: - ValueError: on deserialization issue - - Returns: - Read artifact. - """ - super().load(data_type) - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - import sklearn.ensemble - import sklearn.linear_model - import sklearn.tree - - modules = [sklearn.ensemble, sklearn.linear_model, sklearn.tree] - - with fileio.open(os.path.join(self.uri, "data.json"), "r") as f: - data_json = json.loads(f.read()) - class_name = data_json["model_class"] - cls = None - for module in modules: - if cls := getattr(module, class_name, None): - break - if cls is None: - raise ValueError( - f"Cannot deserialize `{class_name}` using {self.__class__.__name__}. " - f"Only classes from modules {[m.__name__ for m in modules]} " - "are supported" - ) - data = ModelMetadata(cls) - if "search_grid" in data_json: - data.search_grid = data_json["search_grid"] - if "params" in data_json: - data.params = data_json["params"] - if "metric" in data_json: - data.metric = data_json["metric"] - ### YOUR CODE ENDS HERE ### - - return data - - def save(self, data: ModelMetadata) -> None: - """Write to artifact store. - - Args: - data: The data of the artifact to save. - """ - super().save(data) - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - # Dump the model metadata directly into the artifact store as a JSON file - data_json = dict() - with fileio.open(os.path.join(self.uri, "data.json"), "w") as f: - data_json["model_class"] = data.model_class.__name__ - if data.search_grid: - data_json["search_grid"] = {} - for k, v in data.search_grid.items(): - if type(v) == range: - data_json["search_grid"][k] = list(v) - else: - data_json["search_grid"][k] = v - if data.params: - data_json["params"] = data.params - if data.metric: - data_json["metric"] = data.metric - f.write(json.dumps(data_json)) - ### YOUR CODE ENDS HERE ### diff --git a/template/artifacts/model_metadata.py b/template/artifacts/model_metadata.py deleted file mode 100644 index d3e6ea6..0000000 --- a/template/artifacts/model_metadata.py +++ /dev/null @@ -1,62 +0,0 @@ -# {% include 'template/license_header' %} - - -from typing import Any, Dict - -from sklearn.base import ClassifierMixin - - -class ModelMetadata: - """A custom artifact that stores model metadata. - - A model metadata object gathers together information that is collected - about the model being trained in a training pipeline run. This data type - is used for one of the artifacts returned by the model evaluation step. - - This is an example of a *custom artifact data type*: a type returned by - one of the pipeline steps that isn't natively supported by the ZenML - framework. Custom artifact data types are a common occurrence in ZenML, - usually encountered in one of the following circumstances: - - - you use a third party library that is not covered as a ZenML integration - and you model one or more step artifacts from the data types provided by - this library (e.g. datasets, models, data validation profiles, model - evaluation results/reports etc.) - - you need to use one of your own data types as a step artifact and it is - not one of the basic Python artifact data types supported by the ZenML - framework (e.g. str, int, float, dictionaries, lists, etc.) - - you want to extend one of the artifact data types already natively - supported by ZenML (e.g. pandas.DataFrame or sklearn.ClassifierMixin) - to customize it with your own data and/or behavior. - - In all above cases, the ZenML framework lacks one very important piece of - information: it doesn't "know" how to convert the data into a format that - can be saved in the artifact store (e.g. on a filesystem or persistent - storage service like S3 or GCS). Saving and loading artifacts from the - artifact store is something called "materialization" in ZenML terms and - you need to provide this missing information in the form of a custom - materializer - a class that implements loading/saving artifacts from/to - the artifact store. Take a look at the `materializers` folder to see how a - custom materializer is implemented for this artifact data type. - - More information about custom step artifact data types and ZenML - materializers is available in the docs: - - https://docs.zenml.io/user-guide/advanced-guide/artifact-management/handle-custom-data-types - - """ - - ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - def __init__( - self, - model_class: ClassifierMixin, - search_grid: Dict[str, Any] = None, - params: Dict[str, Any] = None, - metric: float = None, - ) -> None: - self.model_class = model_class - self.search_grid = search_grid - self.params = params - self.metric = metric - - ### YOUR CODE ENDS HERE ### From 3fb871403e6d8cdc6492fa9c8784057542eeaaa9 Mon Sep 17 00:00:00 2001 From: Safoine El Khabich <34200873+safoinme@users.noreply.github.com> Date: Fri, 3 Nov 2023 10:53:43 +0100 Subject: [PATCH 18/18] apply reviews --- README.md | 2 +- template/Makefile | 2 - template/gradio/app.py | 43 +++++++++++-------- template/pipelines/deploying.py | 4 +- template/pipelines/training.py | 4 +- template/requirements.txt | 2 +- template/steps/dataset_loader/data_loader.py | 3 +- template/steps/deploying/save_model.py | 11 +++-- ...ace %}huggingface_deployment.py{% endif %} | 1 - .../inference_get_current_version.py | 3 +- template/steps/inference/inference_predict.py | 3 +- template/steps/promotion/promote_latest.py | 5 +-- .../steps/registrer/model_log_register.py | 1 + .../tokenizer_loader/tokenizer_loader.py | 11 +++-- template/utils/misc.py | 7 ++- 15 files changed, 54 insertions(+), 48 deletions(-) diff --git a/README.md b/README.md index 7b6936b..ff26193 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ pip install zenml[templates] Then, run the following command to generate the project: ```bash -zenml init --template nlp-template +zenml init --template template-nlp ``` You will be prompted to provide values for the template parameters. If you want to use the default values, you can add the `--template-with-defaults` flag to the command. diff --git a/template/Makefile b/template/Makefile index f12a621..f88d6f0 100644 --- a/template/Makefile +++ b/template/Makefile @@ -21,5 +21,3 @@ install-local-stack: zenml stack set $${stack_name} && \ zenml stack up -install-remote-stack: - diff --git a/template/gradio/app.py b/template/gradio/app.py index 06d5871..2ec6a09 100644 --- a/template/gradio/app.py +++ b/template/gradio/app.py @@ -1,29 +1,37 @@ # Apache Software License 2.0 -# +# # Copyright (c) ZenML GmbH 2023. All rights reserved. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from os.path import dirname from typing import Optional + import click import numpy as np from transformers import AutoModelForSequenceClassification, AutoTokenizer -from os.path import dirname import gradio as gr + @click.command() -@click.option("--tokenizer_name_or_path", default="tokenizer", help="Name or the path of the tokenizer.") -@click.option("--model_name_or_path", default="model", help="Name or the path of the model.") +@click.option( + "--tokenizer_name_or_path", + default="tokenizer", + help="Name or the path of the tokenizer.", +) +@click.option( + "--model_name_or_path", default="model", help="Name or the path of the model." +) @click.option( "--labels", default="Negative,Positive", help="Comma-separated list of labels." ) @@ -43,23 +51,23 @@ @click.option( "--examples", default="This is an awesome journey, I love it!", - help="Comma-separated list of examples to show in the Gradio interface.", + help="An example to show in the Gradio interface.", ) def sentiment_analysis( - tokenizer_name_or_path: Optional[str], - model_name_or_path: Optional[str], - labels: Optional[str], - title: Optional[str], - description: Optional[str], - interpretation: Optional[str], - examples: Optional[str], + tokenizer_name_or_path: Optional[str], + model_name_or_path: Optional[str], + labels: Optional[str], + title: Optional[str], + description: Optional[str], + interpretation: Optional[str], + examples: Optional[str], ): """Launches a Gradio interface for sentiment analysis. This function launches a Gradio interface for text-classification. It loads a model and a tokenizer from the provided paths and uses them to predict the sentiment of the input text. - + Args: tokenizer_name_or_path (str): Name or the path of the tokenizer. model_name_or_path (str): Name or the path of the model. @@ -71,12 +79,13 @@ def sentiment_analysis( """ labels = labels.split(",") examples = [examples] + def preprocess(text: str) -> str: """Preprocesses the text. Args: text (str): Input text. - + Returns: str: Preprocessed text. """ diff --git a/template/pipelines/deploying.py b/template/pipelines/deploying.py index 2a920c9..8540664 100644 --- a/template/pipelines/deploying.py +++ b/template/pipelines/deploying.py @@ -25,10 +25,10 @@ # Get experiment tracker orchestrator = Client().active_stack.orchestrator -# Check if orchestrator flavor is either default or skypilot +# Check if orchestrator flavor is local if orchestrator.flavor not in ["local"]: raise RuntimeError( - "Your active stack needs to contain a default or skypilot orchestrator for " + "Your active stack needs to contain a local orchestrator for " "the deployment pipeline to work." ) diff --git a/template/pipelines/training.py b/template/pipelines/training.py index 331fd82..502785e 100644 --- a/template/pipelines/training.py +++ b/template/pipelines/training.py @@ -63,9 +63,7 @@ def {{product_name}}_training_pipeline( pipeline_extra = get_pipeline_context().extra ########## Load Dataset stage ########## - dataset = data_loader( - shuffle=True, - ) + dataset = data_loader() ########## Tokenization stage ########## tokenizer = tokenizer_loader( diff --git a/template/requirements.txt b/template/requirements.txt index 0087b4d..3650a1e 100644 --- a/template/requirements.txt +++ b/template/requirements.txt @@ -1,4 +1,4 @@ torchvision accelerate gradio -zenml[server] +zenml[server]==0.45.5 diff --git a/template/steps/dataset_loader/data_loader.py b/template/steps/dataset_loader/data_loader.py index 0604829..9c6cda7 100644 --- a/template/steps/dataset_loader/data_loader.py +++ b/template/steps/dataset_loader/data_loader.py @@ -9,7 +9,6 @@ @step def data_loader( - shuffle: bool = True, ) -> Annotated[DatasetDict, "dataset"]: """ Data loader step. @@ -34,7 +33,7 @@ def data_loader( {%- endif %} {%- if dataset == 'imdb_reviews' %} dataset = load_dataset("imdb")["train"] - dataset = dataset.train_test_split(test_size=0.25, shuffle=shuffle) + dataset = dataset.train_test_split(test_size=0.25, shuffle=True) {%- endif %} {%- if dataset == 'airline_reviews' %} dataset = load_dataset("Shayanvsf/US_Airline_Sentiment") diff --git a/template/steps/deploying/save_model.py b/template/steps/deploying/save_model.py index aaa4e9c..a0c8f76 100644 --- a/template/steps/deploying/save_model.py +++ b/template/steps/deploying/save_model.py @@ -1,12 +1,11 @@ # {% include 'template/license_header' %} -from typing import Optional import mlflow from zenml import step from zenml.client import Client -from zenml.model_registries.base_model_registry import ModelVersionStage from zenml.logger import get_logger +from zenml.model_registries.base_model_registry import ModelVersionStage # Initialize logger logger = get_logger(__name__) @@ -33,7 +32,9 @@ def save_model_to_deploy( stage: The stage of the model in MLFlow. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - logger.info(f" Loading latest version of model {mlflow_model_name} for stage {stage}...") + logger.info( + f" Loading latest version of model {mlflow_model_name} for stage {stage}..." + ) # Load model from MLFlow registry model_version = model_registry.get_latest_model_version( name=mlflow_model_name, @@ -52,5 +53,7 @@ def save_model_to_deploy( # Save model locally transformer_model.model.save_pretrained(model_path) transformer_model.tokenizer.save_pretrained(tokenizer_path) - logger.info(f" Model and tokenizer saved to {model_path} and {tokenizer_path} respectively.") + logger.info( + f" Model and tokenizer saved to {model_path} and {tokenizer_path} respectively." + ) ### YOUR CODE ENDS HERE ### diff --git a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} index d2df257..e1c3a7d 100644 --- a/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} +++ b/template/steps/deploying/{% if deploy_to_huggingface %}huggingface_deployment.py{% endif %} @@ -25,7 +25,6 @@ def deploy_to_huggingface( ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### secret = Client().get_secret("huggingface_creds") assert secret, "No secret found with name 'huggingface_creds'. Please create one that includes your `username` and `token`." - huggingface_username = secret.secret_values["username"] huggingface_token = secret.secret_values["token"] api = HfApi(token=token) hf_repo = api.create_repo(repo_id=repo_name, repo_type="space", space_sdk="gradio", exist_ok=True) diff --git a/template/steps/inference/inference_get_current_version.py b/template/steps/inference/inference_get_current_version.py index 0454d87..a04c2ba 100644 --- a/template/steps/inference/inference_get_current_version.py +++ b/template/steps/inference/inference_get_current_version.py @@ -1,9 +1,8 @@ # {% include 'template/license_header' %} -from typing_extensions import Annotated - from config import MetaConfig +from typing_extensions import Annotated from zenml import step from zenml.client import Client from zenml.logger import get_logger diff --git a/template/steps/inference/inference_predict.py b/template/steps/inference/inference_predict.py index 3406583..851b473 100644 --- a/template/steps/inference/inference_predict.py +++ b/template/steps/inference/inference_predict.py @@ -1,9 +1,8 @@ # {% include 'template/license_header' %} -from typing_extensions import Annotated - import pandas as pd +from typing_extensions import Annotated from zenml import step from zenml.integrations.mlflow.model_deployers.mlflow_model_deployer import ( MLFlowDeploymentService, diff --git a/template/steps/promotion/promote_latest.py b/template/steps/promotion/promote_latest.py index 6b940fe..1a16f53 100644 --- a/template/steps/promotion/promote_latest.py +++ b/template/steps/promotion/promote_latest.py @@ -1,20 +1,19 @@ # {% include 'template/license_header' %} +from config import MetaConfig from zenml import step from zenml.client import Client from zenml.logger import get_logger from zenml.model_registries.base_model_registry import ModelVersionStage -from config import MetaConfig - logger = get_logger(__name__) model_registry = Client().active_stack.model_registry @step -def promote_latest(latest_version:str, current_version:str): +def promote_latest(latest_version: str, current_version: str): """Promote latest trained model. This is an example of a model promotion step, which promotes the diff --git a/template/steps/registrer/model_log_register.py b/template/steps/registrer/model_log_register.py index 3934f4d..eba013c 100644 --- a/template/steps/registrer/model_log_register.py +++ b/template/steps/registrer/model_log_register.py @@ -47,6 +47,7 @@ def register_model( Args: model: The model. tokenizer: The tokenizer. + mlflow_model_name: Name of the model in MLFlow registry. Returns: The trained model and tokenizer. diff --git a/template/steps/tokenizer_loader/tokenizer_loader.py b/template/steps/tokenizer_loader/tokenizer_loader.py index 0ec117e..9f43e30 100644 --- a/template/steps/tokenizer_loader/tokenizer_loader.py +++ b/template/steps/tokenizer_loader/tokenizer_loader.py @@ -1,12 +1,13 @@ # {% include 'template/license_header' %} -from transformers import PreTrainedTokenizerBase, AutoTokenizer +from transformers import AutoTokenizer, PreTrainedTokenizerBase from typing_extensions import Annotated from zenml import step from zenml.logger import get_logger logger = get_logger(__name__) + @step def tokenizer_loader( lower_case: bool, @@ -21,9 +22,9 @@ def tokenizer_loader( This step is parameterized, which allows you to configure the step independently of the step code, before running it in a pipeline. In this example, the step can be configured to use different types of tokenizers corresponding to different - models such as 'bert', 'roberta', or 'distilbert'. + models such as 'bert', 'roberta', or 'distilbert'. - For more information on how to configure steps in a pipeline, refer to the + For more information on how to configure steps in a pipeline, refer to the following documentation: https://docs.zenml.io/user-guide/advanced-guide/configure-steps-pipelines @@ -36,9 +37,7 @@ def tokenizer_loader( The initialized tokenizer. """ ### ADD YOUR OWN CODE HERE - THIS IS JUST AN EXAMPLE ### - tokenizer = AutoTokenizer.from_pretrained( - "{{model}}", do_lower_case=lower_case - ) + tokenizer = AutoTokenizer.from_pretrained("{{model}}", do_lower_case=lower_case) ### YOUR CODE ENDS HERE ### return tokenizer diff --git a/template/utils/misc.py b/template/utils/misc.py index 14abde8..03a64a2 100644 --- a/template/utils/misc.py +++ b/template/utils/misc.py @@ -2,10 +2,9 @@ import numpy as np from datasets import load_metric -from zenml.enums import StrEnum -def compute_metrics(eval_pred: tuple) -> dict[str, float]: +def compute_metrics(eval_pred: tuple[np.ndarray, np.ndarray]) -> dict[str, float]: """Compute the metrics for the model. Args: @@ -32,6 +31,10 @@ def compute_metrics(eval_pred: tuple) -> dict[str, float]: def find_max_length(dataset: list[str]) -> int: """Find the maximum length of the dataset. + The dataset is a list of strings which are the text samples. + We need to find the maximum length of the text samples for + padding. + Args: dataset: The dataset.