From e35cb2601e8fdbaa84341063bf4077dee2e3815d Mon Sep 17 00:00:00 2001 From: Jayesh Sharma Date: Fri, 9 Sep 2022 13:51:24 +0530 Subject: [PATCH] Stack recipe CLI improvements (#872) * fix string indices bug * highlight that pull will delete tfstate files * add check for kubectl, helm and docker * add option to skip checking locals.tf file * update zenml version automatically in recipe * Apply Hamza's suggestions from code review Co-authored-by: Hamza Tahir * Apply Alex's suggestions from code review Co-authored-by: Alex Strick van Linschoten * add filename flag to stack import * Fix failing test * add describe function * Show help message on stack recipe subcommands * check if prerequisites are met * remove pager * add command to check version * Fix docstring and formatting * Add missing return docstring * Fix remaining linting issues after merge * Fix all links * Make returrn type optional Co-authored-by: Hamza Tahir Co-authored-by: Alex Strick van Linschoten Co-authored-by: Michael Schuster --- .gitignore | 3 + .pyspelling-ignore-words | 4 +- .../model-deployers/model-deployers.md | 2 +- .../manual-deployments/aws/aws.md | 20 +- .../manual-deployments/azure/azure.md | 16 +- .../manual-deployments/gcp/gcp-guide.md | 6 +- .../manual-deployments/gcp/gcp.md | 22 +- .../stack-deployment-guide/stack-recipes.md | 2 +- examples/custom_code_deployment/README.md | 6 +- .../README.md | 0 .../__init__.py | 0 .../assets/tekton_ui.png | Bin .../assets/tekton_ui_2.png | Bin .../assets/tekton_ui_3.png | Bin .../assets/zenml.svg | 0 .../config.yaml | 0 .../pipelines/__init__.py | 0 .../pipelines/mnist_pipeline.py | 0 .../run.py | 0 .../steps/__init__.py | 0 .../steps/evaluators.py | 0 .../steps/importers.py | 0 .../steps/normalizers.py | 0 .../steps/trainers.py | 0 src/zenml/cli/stack.py | 5 +- src/zenml/cli/stack_recipes.py | 264 +++++++++++++++--- tests/unit/cli/test_stack.py | 6 +- tests/unit/zen_server/test_zen_server.py | 10 +- 28 files changed, 276 insertions(+), 90 deletions(-) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/README.md (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/__init__.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/assets/tekton_ui.png (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/assets/tekton_ui_2.png (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/assets/tekton_ui_3.png (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/assets/zenml.svg (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/config.yaml (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/pipelines/__init__.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/pipelines/mnist_pipeline.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/run.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/steps/__init__.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/steps/evaluators.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/steps/importers.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/steps/normalizers.py (100%) rename examples/{tekton_pipeline_orchestration => tekton_pipelines_orchestration}/steps/trainers.py (100%) diff --git a/.gitignore b/.gitignore index 081d904387..d7d98e23b5 100644 --- a/.gitignore +++ b/.gitignore @@ -180,3 +180,6 @@ zenml_examples/ # some examples have yamls; don't ignore them !examples/**/*.yaml + +# stack recipes +test_terraform/ \ No newline at end of file diff --git a/.pyspelling-ignore-words b/.pyspelling-ignore-words index 199cd43b21..8d2fd8b64d 100644 --- a/.pyspelling-ignore-words +++ b/.pyspelling-ignore-words @@ -120,6 +120,7 @@ DataFrame's DataFrames DataLoader DataQualityProfileSection +Databricks DatasetProfile DatasetProfileView Deepchecks @@ -734,6 +735,7 @@ daemonization daemonize daemonizer dango +databricks datadir datadrift dataframe @@ -1282,5 +1284,3 @@ zenml zenmldocker zenmlregistry zenserver -databricks -Databricks diff --git a/docs/book/mlops-stacks/model-deployers/model-deployers.md b/docs/book/mlops-stacks/model-deployers/model-deployers.md index 5cfc4c3875..905ded9d0c 100644 --- a/docs/book/mlops-stacks/model-deployers/model-deployers.md +++ b/docs/book/mlops-stacks/model-deployers/model-deployers.md @@ -187,7 +187,7 @@ Both pre- and post-processing are very essential for the model deployment proces The custom model deployment support is available only for the following integrations: * [KServe Custom Predictor](./kserve.md#custom-model-deployment) -* [Seldon Core Custom Python Model](./seldon-core.md#custom-model-deployment) +* [Seldon Core Custom Python Model](./seldon.md#custom-model-deployment) {% endhint %} ### How to Interact with model deployer after deployment? diff --git a/docs/book/stack-deployment-guide/manual-deployments/aws/aws.md b/docs/book/stack-deployment-guide/manual-deployments/aws/aws.md index f75cd198c4..c2b79160d3 100644 --- a/docs/book/stack-deployment-guide/manual-deployments/aws/aws.md +++ b/docs/book/stack-deployment-guide/manual-deployments/aws/aws.md @@ -12,42 +12,42 @@ This is a list of all supported AWS services that you can use as ZenML stack com Amazon Elastic Kubernetes Service (Amazon EKS) is a managed container service to run and scale Kubernetes applications in the cloud or on-premises. [Learn more here](https://docs.aws.amazon.com/eks/latest/userguide/what-is-eks.html). * An EKS cluster can be used to run multiple **orchestrators**. - * [A Kubernetes-native orchestrator.](../../mlops-stacks/orchestrators/kubernetes.md) - * [A Kubeflow orchestrator.](../../mlops-stacks/orchestrators/kubeflow.md) + * [A Kubernetes-native orchestrator.](../../../mlops-stacks/orchestrators/kubernetes.md) + * [A Kubeflow orchestrator.](../../../mlops-stacks/orchestrators/kubeflow.md) * You can host **model deployers** on the cluster. - * [A Seldon model deployer.](../../mlops-stacks/model-deployers/seldon.md) - * [An MLflow model deployer.](../../mlops-stacks/model-deployers/mlflow.md) + * [A Seldon model deployer.](../../../mlops-stacks/model-deployers/seldon.md) + * [An MLflow model deployer.](../../../mlops-stacks/model-deployers/mlflow.md) * Experiment trackers can also be hosted on the cluster. - * [An MLflow experiment tracker](../../mlops-stacks/experiment-trackers/mlflow.md) + * [An MLflow experiment tracker](../../../mlops-stacks/experiment-trackers/mlflow.md) ## Simple Storage Service (S3) Amazon Simple Storage Service (Amazon S3) is an object storage service that offers scalability, data availability, security, and performance. [Learn more here](https://docs.aws.amazon.com/AmazonS3/latest/userguide/Welcome.html). -* You can use an [S3 bucket as an artifact store](../../mlops-stacks/artifact-stores/amazon-s3.md) to hold files from our pipeline runs like models, data and more. +* You can use an [S3 bucket as an artifact store](../../../mlops-stacks/artifact-stores/amazon-s3.md) to hold files from our pipeline runs like models, data and more. ## Elastic Container Registry (ECR) Amazon Elastic Container Registry (Amazon ECR) is an AWS managed container image registry service that is secure, scalable, and reliable. [Learn more here](https://docs.aws.amazon.com/AmazonECR/latest/userguide/what-is-ecr.html). -* An [ECS registry can be used as a container registry](../../mlops-stacks/container-registries/amazon-ecr.md) stack component to host images of your pipelines. +* An [ECS registry can be used as a container registry](../../../mlops-stacks/container-registries/amazon-ecr.md) stack component to host images of your pipelines. ## SageMaker Amazon SageMaker is a fully managed machine learning service. With SageMaker, data scientists and developers can quickly build and train machine learning models, and then directly deploy them into a production-ready hosted environment. [Learn more here](https://docs.aws.amazon.com/sagemaker/latest/dg/whatis.html). -* You can use [SageMaker as a step operator](../../mlops-stacks/step-operators/amazon-sagemaker.md) to run specific steps from your pipeline using it as the backend. +* You can use [SageMaker as a step operator](../../../mlops-stacks/step-operators/amazon-sagemaker.md) to run specific steps from your pipeline using it as the backend. ## Relational Database Service (RDS) Amazon Relational Database Service (Amazon RDS) is a web service that makes it easier to set up, operate, and scale a relational database in the AWS Cloud. [Learn more here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/Welcome.html). -* You can use [Amazon RDS as a metadata store](../../mlops-stacks/metadata-stores/mysql.md) to track metadata from your pipeline runs. +* You can use [Amazon RDS as a metadata store](../../../mlops-stacks/metadata-stores/mysql.md) to track metadata from your pipeline runs. ## Secrets Manager Secrets Manager enables you to replace hardcoded credentials in your code, including passwords, with an API call to Secrets Manager to retrieve the secret programmatically. [Learn more here](https://docs.aws.amazon.com/secretsmanager/latest/userguide/intro.html). -* You can store your secrets to be used inside a pipeline by registering the [AWS Secrets Manager as a ZenML secret manager](../../mlops-stacks/secrets-managers/aws.md) stack component. +* You can store your secrets to be used inside a pipeline by registering the [AWS Secrets Manager as a ZenML secret manager](../../../mlops-stacks/secrets-managers/aws.md) stack component. In the following pages, you will find step-by-step guides for setting up some common stacks using the AWS console and the CLI. More combinations and components are progressively updated in the form of new pages. diff --git a/docs/book/stack-deployment-guide/manual-deployments/azure/azure.md b/docs/book/stack-deployment-guide/manual-deployments/azure/azure.md index c51c431b63..30b464370a 100644 --- a/docs/book/stack-deployment-guide/manual-deployments/azure/azure.md +++ b/docs/book/stack-deployment-guide/manual-deployments/azure/azure.md @@ -13,31 +13,31 @@ Azure Kubernetes Service (AKS) is a managed Kubernetes service with hardened sec * An AKS cluster can be used to run multiple **orchestrators**. - * [A Kubernetes-native orchestrator.](../../mlops-stacks/orchestrators/kubernetes.md) - * [A Kubeflow orchestrator.](../../mlops-stacks/orchestrators/kubeflow.md) + * [A Kubernetes-native orchestrator.](../../../mlops-stacks/orchestrators/kubernetes.md) + * [A Kubeflow orchestrator.](../../../mlops-stacks/orchestrators/kubeflow.md) * You can host **model deployers** on the cluster. - * [A Seldon model deployer.](../../mlops-stacks/model-deployers/seldon.md) - * [An MLflow model deployer.](../../mlops-stacks/model-deployers/mlflow.md) + * [A Seldon model deployer.](../../../mlops-stacks/model-deployers/seldon.md) + * [An MLflow model deployer.](../../../mlops-stacks/model-deployers/mlflow.md) * Experiment trackers can also be hosted on the cluster. - * [An MLflow experiment tracker](../../mlops-stacks/model-deployers/mlflow.md) + * [An MLflow experiment tracker](../../../mlops-stacks/model-deployers/mlflow.md) ## Azure Blob Storage Azure Blob storage is Microsoft's object storage solution for the cloud. Blob storage is optimized for storing massive amounts of unstructured data. Blob storage offers three types of resources: the storage account, a container in the storage account and a blob in a container. [Learn more here](https://docs.microsoft.com/en-us/azure/storage/blobs/storage-blobs-introduction). -* You can use an [Azure Blob Storage Container as an artifact store](../../mlops-stacks/artifact-stores/azure-blob-storage.md) to hold files from our pipeline runs like models, data and more. +* You can use an [Azure Blob Storage Container as an artifact store](../../../mlops-stacks/artifact-stores/azure-blob-storage.md) to hold files from our pipeline runs like models, data and more. ## Azure Container Registry Azure Container Registry is a managed registry service based on the open-source Docker Registry 2.0. Create and maintain Azure container registries to store and manage your container images and related artifacts. [Learn more here](https://docs.microsoft.com/en-us/azure/container-registry/container-registry-intro). -* An [Azure container registry can be used as a ZenML container registry](../../mlops-stacks/container-registries/azure.md) stack component to host images of your pipelines. +* An [Azure container registry can be used as a ZenML container registry](../../../mlops-stacks/container-registries/azure.md) stack component to host images of your pipelines. ## AzureML Azure Machine Learning is a cloud service for accelerating and managing the machine learning project lifecycle. Machine learning professionals, data scientists, and engineers can use it in their day-to-day workflows to train and deploy models, and manage MLOps. [Learn more here](https://docs.microsoft.com/en-us/azure/machine-learning/overview-what-is-azure-machine-learning). -* You can use [AzureML compute as a step operator](../../mlops-stacks/step-operators/azureml.md) to run specific steps from your pipeline using it as the backend. +* You can use [AzureML compute as a step operator](../../../mlops-stacks/step-operators/azureml.md) to run specific steps from your pipeline using it as the backend. ## Azure SQL server diff --git a/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp-guide.md b/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp-guide.md index acbf3f11bc..242b4b033c 100644 --- a/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp-guide.md +++ b/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp-guide.md @@ -430,7 +430,7 @@ appropriate rights after the first run fails. At the end of the logs you should be seeing a link to the Vertex AI dashboard. It should look something like this: -![Finished Run](../../assets/VertexAiRun.png) +![Finished Run](../../../assets/VertexAiRun.png) In case you get an error message like this: ```shell @@ -473,8 +473,8 @@ Now rerun your pipeline, it should work now. Within this guide you have set up and used a stack on GCP using the Vertex AI orchestrator. For more guides on different cloud set-ups, check out the -[Kubeflow](../../mlops-stacks/orchestrators/kubeflow.md) and -[Kubernetes](../../mlops-stacks/orchestrators/kubernetes.md) orchestrators +[Kubeflow](../../../mlops-stacks/orchestrators/kubeflow.md) and +[Kubernetes](../../../mlops-stacks/orchestrators/kubernetes.md) orchestrators respectively and find out if these are a better fit for you. diff --git a/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp.md b/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp.md index 360ab69015..efd5d16ed5 100644 --- a/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp.md +++ b/docs/book/stack-deployment-guide/manual-deployments/gcp/gcp.md @@ -13,45 +13,45 @@ Google Kubernetes Engine (GKE) provides a managed environment for deploying, man * An GKE cluster can be used to run multiple **orchestrators**. - * [A Kubernetes-native orchestrator.](../../mlops-stacks/orchestrators/kubernetes.md) - * [A Kubeflow orchestrator.](../../mlops-stacks/orchestrators/kubeflow.md) + * [A Kubernetes-native orchestrator.](../../../mlops-stacks/orchestrators/kubernetes.md) + * [A Kubeflow orchestrator.](../../../mlops-stacks/orchestrators/kubeflow.md) * You can host **model deployers** on the cluster. - * [A Seldon model deployer.](../../mlops-stacks/model-deployers/seldon.md) - * [An MLflow model deployer.](../../mlops-stacks/model-deployers/mlflow.md) + * [A Seldon model deployer.](../../../mlops-stacks/model-deployers/seldon.md) + * [An MLflow model deployer.](../../../mlops-stacks/model-deployers/mlflow.md) * Experiment trackers can also be hosted on the cluster. - * [An MLflow experiment tracker](../../mlops-stacks/experiment-trackers/mlflow.md) + * [An MLflow experiment tracker](../../../mlops-stacks/experiment-trackers/mlflow.md) ## Cloud Storage Bucket (GCS) Cloud Storage is a service for storing your objects in Google Cloud. An object is an immutable piece of data consisting of a file of any format. You store objects in containers called buckets. [Learn more here](https://cloud.google.com/storage/docs/introduction). -* You can use a [GCS bucket as an artifact store](../../mlops-stacks/artifact-stores/gcloud-gcs.md) to hold files from our pipeline runs like models, data and more. +* You can use a [GCS bucket as an artifact store](../../../mlops-stacks/artifact-stores/gcloud-gcs.md) to hold files from our pipeline runs like models, data and more. ## Google Container Registry (GCR) Container Registry is a service for storing private container images. It is being deprecated in favor of Artifact Registry, support for which will be coming soon to ZenML! -* A [GCR registry can be used as a container registry](../../mlops-stacks/container-registries/gcloud.md) stack component to host images of your pipelines. +* A [GCR registry can be used as a container registry](../../../mlops-stacks/container-registries/gcloud.md) stack component to host images of your pipelines. ## Vertex AI Vertex AI brings together the Google Cloud services for building ML under one, unified UI and API. In Vertex AI, you can now train and compare models using AutoML or custom code training and all your models are stored in one central model repository. [Learn more here](https://cloud.google.com/vertex-ai). -* You can use [Vertex AI as a step operator](../../mlops-stacks/step-operators/gcloud-vertexai.md) to run specific steps from your pipeline using it as the backend. +* You can use [Vertex AI as a step operator](../../../mlops-stacks/step-operators/gcloud-vertexai.md) to run specific steps from your pipeline using it as the backend. -* [Vertex AI can also be used as an orchestrator](../../mlops-stacks/orchestrators/gcloud-vertexai.md) for your pipelines. +* [Vertex AI can also be used as an orchestrator](../../../mlops-stacks/orchestrators/gcloud-vertexai.md) for your pipelines. ## CloudSQL Cloud SQL is a fully-managed database service that helps you set up, maintain, manage, and administer your relational databases on Google Cloud Platform. You can use Cloud SQL with a MySQL server in ZenML. [Learn more here](https://cloud.google.com/sql/docs). -* You can use a [CloudSQL MySQL instance as a metadata store](../../mlops-stacks/metadata-stores/mysql.md) to track metadata from your pipeline runs. +* You can use a [CloudSQL MySQL instance as a metadata store](../../../mlops-stacks/metadata-stores/mysql.md) to track metadata from your pipeline runs. ## Secret Manager Secret Manager is a secure and convenient storage system for API keys, passwords, certificates, and other sensitive data. Secret Manager provides a central place and single source of truth to manage, access, and audit secrets across Google Cloud. [Learn more here](https://cloud.google.com/secret-manager/docs). -* You can store your secrets to be used inside a pipeline by registering the [Google Secret Manager as a ZenML secret manager](../../mlops-stacks/secrets-managers/gcp.md) stack component. +* You can store your secrets to be used inside a pipeline by registering the [Google Secret Manager as a ZenML secret manager](../../../mlops-stacks/secrets-managers/gcp.md) stack component. In the following pages, you will find step-by-step guides for setting up some common stacks using the GCP console and the CLI. More combinations and components are progressively updated in the form of new pages. diff --git a/docs/book/stack-deployment-guide/stack-recipes.md b/docs/book/stack-deployment-guide/stack-recipes.md index baba89ef66..f6d5618065 100644 --- a/docs/book/stack-deployment-guide/stack-recipes.md +++ b/docs/book/stack-deployment-guide/stack-recipes.md @@ -19,7 +19,7 @@ To use the stack recipe CLI commands, you will have to install some optional dep Run `pip install "zenml[stacks]"` to get started! {% endhint %} -Detailed steps are available in the READMEs of respective recipes but here's what a simple flow could look like: +Detailed steps are available in the README of the respective recipe but here's what a simple flow could look like: 1. 📃 List the available recipes in the repository. diff --git a/examples/custom_code_deployment/README.md b/examples/custom_code_deployment/README.md index 0de40bc581..efa7c1813d 100644 --- a/examples/custom_code_deployment/README.md +++ b/examples/custom_code_deployment/README.md @@ -172,7 +172,7 @@ The next sections cover how to setup the GCP Artifact Store credentials for the Please look up the variables relevant to your use case in the [official KServe Storage Credentials](https://kserve.github.io/website/0.8/sdk_docs/docs/KServeClient/#parameters) and set them accordingly for your ZenML secrets schemas already built for each storage_type. -You can find the relevant variables in the [Kserve integration secret schemas docs](https://apidocs.zenml.io/0.13.0/api_docs/integrations/#zenml.integrations.kserve.secret_schemas.secret_schemas). +You can find the relevant variables in the [Kserve integration secret schemas docs](https://apidocs.zenml.io/latest/api_docs/integrations/#zenml.integrations.kserve.secret_schemas.secret_schemas). #### GCP Authentication with kserve_gs secret schema @@ -389,7 +389,7 @@ The next sections cover how to set GCP Artifact Store credentials for the Seldon Please look up the variables relevant to your use case in the [official Seldon Core Storage Credentials](https://kserve.github.io/website/0.8/sdk_docs/docs/KServeClient/#parameters) and set them accordingly for your ZenML secrets schemas already built for each storage_type. -You can find the relevant variables in the [Seldon Integration secret schema](https://apidocs.zenml.io/0.13.0/api_docs/integrations/#zenml.integrations.seldon.secret_schemas.secret_schemas). +You can find the relevant variables in the [Seldon Integration secret schema](https://apidocs.zenml.io/latest/api_docs/integrations/#zenml.integrations.seldon.secret_schemas.secret_schemas). #### GCP Authentication with seldon_s3 secret schema @@ -621,7 +621,7 @@ rm -rf zenml_examples # 📜 Learn more -Our docs regarding the custom model deployment can be found [here](https://docs.zenml.io/mlops-stacks/model-deployers/custom-pre-processing-and-post-processing). +Our docs regarding the custom model deployment can be found [here](https://docs.zenml.io/mlops-stacks/model-deployers#custom-pre-processing-and-post-processing). If you want to learn more about the deployment in ZenML in general or about how to build your deployer steps in ZenML check out our [docs](https://docs.zenml.io/mlops-stacks/model-deployers/custom). diff --git a/examples/tekton_pipeline_orchestration/README.md b/examples/tekton_pipelines_orchestration/README.md similarity index 100% rename from examples/tekton_pipeline_orchestration/README.md rename to examples/tekton_pipelines_orchestration/README.md diff --git a/examples/tekton_pipeline_orchestration/__init__.py b/examples/tekton_pipelines_orchestration/__init__.py similarity index 100% rename from examples/tekton_pipeline_orchestration/__init__.py rename to examples/tekton_pipelines_orchestration/__init__.py diff --git a/examples/tekton_pipeline_orchestration/assets/tekton_ui.png b/examples/tekton_pipelines_orchestration/assets/tekton_ui.png similarity index 100% rename from examples/tekton_pipeline_orchestration/assets/tekton_ui.png rename to examples/tekton_pipelines_orchestration/assets/tekton_ui.png diff --git a/examples/tekton_pipeline_orchestration/assets/tekton_ui_2.png b/examples/tekton_pipelines_orchestration/assets/tekton_ui_2.png similarity index 100% rename from examples/tekton_pipeline_orchestration/assets/tekton_ui_2.png rename to examples/tekton_pipelines_orchestration/assets/tekton_ui_2.png diff --git a/examples/tekton_pipeline_orchestration/assets/tekton_ui_3.png b/examples/tekton_pipelines_orchestration/assets/tekton_ui_3.png similarity index 100% rename from examples/tekton_pipeline_orchestration/assets/tekton_ui_3.png rename to examples/tekton_pipelines_orchestration/assets/tekton_ui_3.png diff --git a/examples/tekton_pipeline_orchestration/assets/zenml.svg b/examples/tekton_pipelines_orchestration/assets/zenml.svg similarity index 100% rename from examples/tekton_pipeline_orchestration/assets/zenml.svg rename to examples/tekton_pipelines_orchestration/assets/zenml.svg diff --git a/examples/tekton_pipeline_orchestration/config.yaml b/examples/tekton_pipelines_orchestration/config.yaml similarity index 100% rename from examples/tekton_pipeline_orchestration/config.yaml rename to examples/tekton_pipelines_orchestration/config.yaml diff --git a/examples/tekton_pipeline_orchestration/pipelines/__init__.py b/examples/tekton_pipelines_orchestration/pipelines/__init__.py similarity index 100% rename from examples/tekton_pipeline_orchestration/pipelines/__init__.py rename to examples/tekton_pipelines_orchestration/pipelines/__init__.py diff --git a/examples/tekton_pipeline_orchestration/pipelines/mnist_pipeline.py b/examples/tekton_pipelines_orchestration/pipelines/mnist_pipeline.py similarity index 100% rename from examples/tekton_pipeline_orchestration/pipelines/mnist_pipeline.py rename to examples/tekton_pipelines_orchestration/pipelines/mnist_pipeline.py diff --git a/examples/tekton_pipeline_orchestration/run.py b/examples/tekton_pipelines_orchestration/run.py similarity index 100% rename from examples/tekton_pipeline_orchestration/run.py rename to examples/tekton_pipelines_orchestration/run.py diff --git a/examples/tekton_pipeline_orchestration/steps/__init__.py b/examples/tekton_pipelines_orchestration/steps/__init__.py similarity index 100% rename from examples/tekton_pipeline_orchestration/steps/__init__.py rename to examples/tekton_pipelines_orchestration/steps/__init__.py diff --git a/examples/tekton_pipeline_orchestration/steps/evaluators.py b/examples/tekton_pipelines_orchestration/steps/evaluators.py similarity index 100% rename from examples/tekton_pipeline_orchestration/steps/evaluators.py rename to examples/tekton_pipelines_orchestration/steps/evaluators.py diff --git a/examples/tekton_pipeline_orchestration/steps/importers.py b/examples/tekton_pipelines_orchestration/steps/importers.py similarity index 100% rename from examples/tekton_pipeline_orchestration/steps/importers.py rename to examples/tekton_pipelines_orchestration/steps/importers.py diff --git a/examples/tekton_pipeline_orchestration/steps/normalizers.py b/examples/tekton_pipelines_orchestration/steps/normalizers.py similarity index 100% rename from examples/tekton_pipeline_orchestration/steps/normalizers.py rename to examples/tekton_pipelines_orchestration/steps/normalizers.py diff --git a/examples/tekton_pipeline_orchestration/steps/trainers.py b/examples/tekton_pipelines_orchestration/steps/trainers.py similarity index 100% rename from examples/tekton_pipeline_orchestration/steps/trainers.py rename to examples/tekton_pipelines_orchestration/steps/trainers.py diff --git a/src/zenml/cli/stack.py b/src/zenml/cli/stack.py index 5f76924d9c..1276f32b81 100644 --- a/src/zenml/cli/stack.py +++ b/src/zenml/cli/stack.py @@ -1142,7 +1142,7 @@ def _import_stack_component( @stack.command("import", help="Import a stack from YAML.") @click.argument("stack_name", type=str, required=True) -@click.argument("filename", type=str, required=False) +@click.option("--filename", "-f", type=str, required=False) @click.option( "--ignore-version-mismatch", is_flag=True, @@ -1150,8 +1150,7 @@ def _import_stack_component( "is different from the one specified in the stack YAML file", ) @click.option( - "--decouple_stores", - "decouple_stores", + "--decouple-stores", is_flag=True, help="Decouple the given artifact/metadata store from prior associations.", type=click.BOOL, diff --git a/src/zenml/cli/stack_recipes.py b/src/zenml/cli/stack_recipes.py index fa84f26929..3e952a44e2 100644 --- a/src/zenml/cli/stack_recipes.py +++ b/src/zenml/cli/stack_recipes.py @@ -22,18 +22,16 @@ import click from packaging.version import Version, parse -from rich.markdown import Markdown from rich.text import Text import zenml from zenml.cli import utils as cli_utils from zenml.cli.cli import TagGroup from zenml.cli.stack import import_stack, stack -from zenml.console import console from zenml.exceptions import GitNotFoundError from zenml.io import fileio from zenml.logger import get_logger -from zenml.utils import io_utils +from zenml.utils import io_utils, yaml_utils from zenml.utils.analytics_utils import AnalyticsEvent, track_event logger = get_logger(__name__) @@ -48,7 +46,12 @@ "through these commands. If you encounter any problems, create an issue " f"on the repository {STACK_RECIPES_GITHUB_REPO} and we'll help you out!" ) -HELP_MESSAGE = "Commands for using the stack recipes." +NOT_INSTALLED_MESSAGE = ( + "The stack recipe commands seem to be unavailable on your machine. This " + "is probably because ZenML was installed without the optional terraform " + "dependencies. To install the missing dependencies: \n\n" + f'`pip install "zenml[stacks]=={zenml.__version__}"`.' +) try: # Make sure all stack recipe dependencies are installed @@ -61,13 +64,6 @@ # just fail. terraform_installed = False - HELP_MESSAGE += ( - "\n\n**Note**: The stack recipe commands seem to be unavailable on " - "your machine. This is probably because ZenML was installed without " - "the optional terraform dependencies. To install the missing dependencies: \n\n" - f'`pip install "zenml[stacks]=={zenml.__version__}"`.' - ) - class Terraform: """Class to represent terraform applications.""" @@ -81,28 +77,97 @@ def __init__(self, path: str) -> None: self.tf = python_terraform.Terraform(working_dir=str(path)) def check_installation(self) -> None: - """Checks if terraform is installed on the host system. + """Checks if necessary tools are installed on the host system. Raises: - RuntimeError: if terraform is not installed. + RuntimeError: if any required tool is not installed. """ - if not self._is_installed(): + if not self._is_terraform_installed(): raise RuntimeError( "Terraform is required for stack recipes to run and was not " - "found installed on your machine. Please visit " + "found installed on your machine or not available on " + "your $PATH. Please visit " "https://learn.hashicorp.com/tutorials/terraform/install-cli " "to install it." ) + if not self._is_kubectl_installed(): + raise RuntimeError( + "kubectl is not installed on your machine or not available on " + "your $PATH. It is used by stack recipes to create some " + "resources on Kubernetes and to configure access to your " + "cluster. Please visit " + "https://kubernetes.io/docs/tasks/tools/#kubectl " + "to install it." + ) + if not self._is_helm_installed(): + raise RuntimeError( + "Helm is not installed on your machine or not available on " + "your $PATH. It is required for stack recipes to create releases " + "on Kubernetes. Please visit " + "https://helm.sh/docs/intro/install/ " + "to install it." + ) + if not self._is_docker_installed(): + raise RuntimeError( + "Docker is not installed on your machine or not available on " + "your $PATH. It is required for stack recipes to configure " + "access to the container registry. Please visit " + "https://docs.docker.com/engine/install/ " + "to install it." + ) - def _is_installed(self) -> bool: + def _is_terraform_installed(self) -> bool: """Checks if terraform is installed on the host system. Returns: True if terraform is installed, false otherwise. """ # check terraform version to verify installation. - ret_code, _, _ = self.tf.cmd("-version") - return bool(ret_code == 0) + try: + self.tf.cmd("-version") + except FileNotFoundError: + return False + + return True + + def _is_kubectl_installed(self) -> bool: + """Checks if kubectl is installed on the host system. + + Returns: + True if kubectl is installed, false otherwise. + """ + try: + subprocess.check_output(["kubectl"]) + except subprocess.CalledProcessError: + return False + + return True + + def _is_helm_installed(self) -> bool: + """Checks if helm is installed on the host system. + + Returns: + True if helm is installed, false otherwise. + """ + try: + subprocess.check_output(["helm", "version"]) + except subprocess.CalledProcessError: + return False + + return True + + def _is_docker_installed(self) -> bool: + """Checks if docker is installed on the host system. + + Returns: + True if docker is installed, false otherwise. + """ + try: + subprocess.check_output(["docker", "--version"]) + except subprocess.CalledProcessError: + return False + + return True def apply(self) -> str: """Function to call terraform init and terraform apply. @@ -141,7 +206,9 @@ def apply(self) -> str: ) # return the path of the stack yaml file - _, stack_file_path, _ = self.tf.output("stack-yaml-path") + _, stack_file_path, _ = self.tf.output( + "stack-yaml-path", full_value=True + ) return str(stack_file_path) def _get_vars(self, path: str) -> Any: @@ -171,6 +238,9 @@ def _get_vars(self, path: str) -> Any: # read values into a dict and return with fileio.open(variables_file_path, "r") as f: variables = json.load(f) + + # update zenml-version variable to current version + variables["zenml-version"] = zenml.__version__ return variables def set_log_level(self, log_level: str) -> None: @@ -192,7 +262,7 @@ def destroy(self) -> None: class LocalStackRecipe: - """Class to encapsulate the local stack that can be run from the CLI.""" + """Class to encapsulate the local recipe that can be run from the CLI.""" def __init__(self, path: Path, name: str) -> None: """Create a new LocalStack instance. @@ -472,7 +542,7 @@ def is_stack_recipe(self, stack_recipe_name: Optional[str] = None) -> bool: def get_stack_recipes( self, stack_recipe_name: Optional[str] = None ) -> List[StackRecipe]: - """Method that allows you to get an stack recipe by name. + """Method that allows you to get a stack recipe by name. If no stack recipe is supplied, all stack recipes are returned. @@ -556,6 +626,15 @@ def clean_current_stack_recipes(self) -> None: ) shutil.rmtree(stack_recipes_directory) + def get_active_version(self) -> Optional[str]: + """Returns the active version of the mlops-stacks repository. + + Returns: + The active version of the repository. + """ + self.stack_recipe_repo.checkout_latest_release() + return self.stack_recipe_repo.active_version + pass_git_stack_recipes_handler = click.make_pass_decorator( GitStackRecipesHandler, ensure=True @@ -563,7 +642,34 @@ def clean_current_stack_recipes(self) -> None: pass_tf_client = click.make_pass_decorator(Terraform, ensure=True) -@stack.group("recipe", cls=TagGroup, help=HELP_MESSAGE) +class RecipeGroup(TagGroup): + """Click group that always prints a warning message.""" + + def invoke(self, ctx: click.Context) -> Any: + """Invokes the subcommand or prints an error message. + + If terraform is installed, this forwards the invocation to the super + class which invokes the subcommand. If terraform is not installed, + prints a warning message. + + Args: + ctx: Click context. + + Returns: + Invocation result. + """ + if terraform_installed: + return super().invoke(ctx) + else: + cli_utils.warning(NOT_INSTALLED_MESSAGE) + + +@stack.group( + "recipe", + cls=RecipeGroup, + help="Commands for using the stack recipes.", + invoke_without_command=True, +) def stack_recipe() -> None: """Access all ZenML stack recipes.""" @@ -657,9 +763,58 @@ def info( cli_utils.error(str(e)) else: - md = Markdown(stack_recipe_obj.readme_content) - with console.pager(styles=True): - console.print(md) + print(stack_recipe_obj.readme_content) + + @stack_recipe.command( + help="Describe the stack components and their tools that are " + "created as part of this recipe." + ) + @pass_git_stack_recipes_handler + @click.argument("stack_recipe_name") + def describe( + git_stack_recipes_handler: GitStackRecipesHandler, + stack_recipe_name: str, + ) -> None: + """Describe the stack components and their tools that are created as part of this recipe. + + Outputs a the "Description" section of the recipe metadata. + + Args: + git_stack_recipes_handler: The GitStackRecipesHandler instance. + stack_recipe_name: The name of the stack recipe. + """ + try: + stack_recipe_obj = git_stack_recipes_handler.get_stack_recipes( + stack_recipe_name + )[0] + except KeyError as e: + cli_utils.error(str(e)) + + else: + metadata = yaml_utils.read_yaml( + file_path=os.path.join( + stack_recipe_obj.path_in_repo, "metadata.yaml" + ) + ) + logger.info(metadata["Description"]) + + @stack_recipe.command( + help="The active version of the mlops-stacks repository" + ) + @pass_git_stack_recipes_handler + def version( + git_stack_recipes_handler: GitStackRecipesHandler, + ) -> None: + """The active version of the mlops-stacks repository. + + Args: + git_stack_recipes_handler: The GitStackRecipesHandler instance. + """ + active_version = git_stack_recipes_handler.get_active_version() + if active_version: + cli_utils.declare(active_version) + else: + cli_utils.warning("Unable to detect version.") @stack_recipe.command( help="Pull stack recipes straight into your current working directory." @@ -721,9 +876,13 @@ def pull( name=stack_recipe.name, path=Path(destination_dir) ).is_present(): if force or cli_utils.confirmation( - f"Stack recipe {stack_recipe.name} is already pulled. " - "Do you wish to overwrite the directory at " - f"{destination_dir}?" + f"Stack recipe {stack_recipe.name} is already pulled at " + f"{destination_dir}." + "\nOverwriting this directory will delete all terraform " + "state files and the local configuration. We recommend " + "that you do this only once the remote resources have " + "been destroyed." + "Do you wish to proceed with overwriting?" ): fileio.rmtree(destination_dir) else: @@ -770,9 +929,9 @@ def pull( "-f", "force", is_flag=True, - help="Force the run of the stack_recipe. This deletes the .zen folder from the " - "stack_recipe folder and force installs all necessary integration " - "requirements.", + help="Force pull the stack recipe. This overwrites any existing recipe " + "files present locally, including the terraform state files and the " + "local configuration.", ) @click.option( "--stack-name", @@ -798,6 +957,12 @@ def pull( "log level for the deploy operation.", default="ERROR", ) + @click.option( + "--skip-check", + "-s", + is_flag=True, + help="Skip the checking of locals.tf file before executing the recipe.", + ) @pass_git_stack_recipes_handler @click.pass_context def deploy( @@ -808,6 +973,7 @@ def deploy( force: bool, import_stack_flag: bool, log_level: str, + skip_check: bool, stack_name: Optional[str], ) -> None: """Run the stack_recipe at the specified relative path. @@ -820,7 +986,7 @@ def deploy( git_stack_recipes_handler: The GitStackRecipesHandler instance. stack_recipe_name: The name of the stack_recipe. path: The path at which you want to install the stack_recipe(s). - force: Force the run of the stack_recipe. + force: Force pull the stack recipe, overwriting any existing files. stack_name: A name for the ZenML stack that gets imported as a result of the recipe deployment. import_stack_flag: Import the stack automatically after the recipe is @@ -828,6 +994,7 @@ def deploy( can be imported manually otherwise. log_level: Choose one of TRACE, DEBUG, INFO, WARN or ERROR (case insensitive) as log level for the deploy operation. + skip_check: Skip the checking of locals.tf file before executing the recipe. """ cli_utils.warning(ALPHA_MESSAGE) stack_recipes_dir = Path(os.getcwd()) / path @@ -871,17 +1038,34 @@ def deploy( # set terraform log level tf_client.set_log_level(log_level=log_level) - logger.info( - "The following values are selected for the configuration " - "of your cloud resources. You can change it by modifying " - "the contents of the locals.tf file here: " - f"{os.path.join(local_stack_recipe.path, 'locals.tf')}\n" + # warn that prerequisites should be met + metadata = yaml_utils.read_yaml( + file_path=os.path.join( + local_stack_recipe.path, "metadata.yaml" + ) ) + if not cli_utils.confirmation( + "\nPrerequisites for running this recipe are as follows.\n" + f"{metadata['Prerequisites']}" + "\n\n Are all of these conditions met?" + ): + cli_utils.warning( + "Prerequisites are not installed. Please make sure they " + "are met and run deploy again." + ) + return + + if not skip_check: + logger.info( + "The following values are selected for the configuration " + "of your cloud resources. You can change it by modifying " + "the contents of the locals.tf file here: " + f"{os.path.join(local_stack_recipe.path, 'locals.tf')}\n" + ) - with console.pager(styles=False): - console.print(local_stack_recipe.locals_content) + print(local_stack_recipe.locals_content) - if cli_utils.confirmation( + if skip_check or cli_utils.confirmation( f"\nDo you wish to deploy the {stack_recipe_name} recipe " "with the above configuration? Please make sure that " "resources with the same values as above don't already " diff --git a/tests/unit/cli/test_stack.py b/tests/unit/cli/test_stack.py index 5c478e0fe5..f1f2e0ef31 100644 --- a/tests/unit/cli/test_stack.py +++ b/tests/unit/cli/test_stack.py @@ -261,8 +261,8 @@ def test_stack_export_delete_import(clean_repo) -> None: # export stack runner = CliRunner() - export_import_args = ["arias_new_stack", "arias_new_stack.yaml"] - result = runner.invoke(export_stack, export_import_args) + file_name = "arias_new_stack.yaml" + result = runner.invoke(export_stack, [stack_name, file_name]) assert result.exit_code == 0 assert os.path.exists("arias_new_stack.yaml") @@ -281,7 +281,7 @@ def test_stack_export_delete_import(clean_repo) -> None: clean_repo.get_stack("arias_new_stack") # import stack - result = runner.invoke(import_stack, export_import_args) + result = runner.invoke(import_stack, [stack_name, "--filename", file_name]) assert result.exit_code == 0 assert clean_repo.get_stack("arias_new_stack") is not None diff --git a/tests/unit/zen_server/test_zen_server.py b/tests/unit/zen_server/test_zen_server.py index f27680bcf1..c0e6a96673 100644 --- a/tests/unit/zen_server/test_zen_server.py +++ b/tests/unit/zen_server/test_zen_server.py @@ -25,8 +25,8 @@ from zenml.zen_stores import LocalZenStore from zenml.zen_stores.base_zen_store import DEFAULT_USERNAME +SERVER_START_STOP_TIMEOUT = 15 -TIMEOUT = 15 @pytest.fixture def running_zen_server(tmp_path_factory: pytest.TempPathFactory) -> ZenServer: @@ -47,10 +47,10 @@ def running_zen_server(tmp_path_factory: pytest.TempPathFactory) -> ZenServer: ZenServerConfig(port=port, profile_name=store_profile.name) ) - zen_server.start(timeout=TIMEOUT) + zen_server.start(timeout=SERVER_START_STOP_TIMEOUT) yield zen_server - zen_server.stop(timeout=TIMEOUT) + zen_server.stop(timeout=SERVER_START_STOP_TIMEOUT) global_cfg.delete_profile(store_profile.name) assert zen_server.check_status()[0] == ServiceState.INACTIVE @@ -110,10 +110,10 @@ def test_server_up_down(): ) endpoint = f"http://127.0.0.1:{port}/" try: - zen_server.start(timeout=TIMEOUT) + zen_server.start(timeout=SERVER_START_STOP_TIMEOUT) assert zen_server.check_status()[0] == ServiceState.ACTIVE assert zen_server.endpoint.status.uri == endpoint assert requests.head(endpoint + "health").status_code == 200 finally: - zen_server.stop(timeout=TIMEOUT) + zen_server.stop(timeout=SERVER_START_STOP_TIMEOUT) assert zen_server.check_status()[0] == ServiceState.INACTIVE