diff --git a/site/config.yaml b/site/config.yaml index 85b228d05..bb9eff743 100644 --- a/site/config.yaml +++ b/site/config.yaml @@ -23,15 +23,16 @@ params: ytt: name: ytt root_link: /ytt/ - latest_docs_link: /ytt/docs/v0.49.x/ + latest_docs_link: /ytt/docs/v0.50.x/ github_url: https://github.com/carvel-dev/ytt search: true search_index_name: carvel-ytt search_api_key: a38560864c2e9128ae57d5734df438ff versioning: true - version_latest: v0.49.x + version_latest: v0.50.x versions: - develop + - v0.50.x - v0.49.x - v0.48.0 - v0.47.x diff --git a/site/content/ytt/docs/v0.49.x/_index.md b/site/content/ytt/docs/v0.49.x/_index.md index a15c8732b..dae24809b 100644 --- a/site/content/ytt/docs/v0.49.x/_index.md +++ b/site/content/ytt/docs/v0.49.x/_index.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/] + title: "About ytt" cascade: version: v0.49.x diff --git a/site/content/ytt/docs/v0.49.x/data-values-schema-migration-guide.md b/site/content/ytt/docs/v0.49.x/data-values-schema-migration-guide.md index f272646f9..12cee5cc4 100644 --- a/site/content/ytt/docs/v0.49.x/data-values-schema-migration-guide.md +++ b/site/content/ytt/docs/v0.49.x/data-values-schema-migration-guide.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/data-values-schema-migration-guide] + title: Schema Migration Guide toc: "true" --- diff --git a/site/content/ytt/docs/v0.49.x/data-values-vs-overlays.md b/site/content/ytt/docs/v0.49.x/data-values-vs-overlays.md index 78353da0c..e81b9bbfe 100644 --- a/site/content/ytt/docs/v0.49.x/data-values-vs-overlays.md +++ b/site/content/ytt/docs/v0.49.x/data-values-vs-overlays.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/data-values-vs-overlays] + title: "Data Values vs Overlays" --- diff --git a/site/content/ytt/docs/v0.49.x/faq.md b/site/content/ytt/docs/v0.49.x/faq.md index 09f859583..ea0a1ad96 100644 --- a/site/content/ytt/docs/v0.49.x/faq.md +++ b/site/content/ytt/docs/v0.49.x/faq.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/faq] + title: FAQ --- diff --git a/site/content/ytt/docs/v0.49.x/file-marks.md b/site/content/ytt/docs/v0.49.x/file-marks.md index 4f87f2e7f..43652038a 100644 --- a/site/content/ytt/docs/v0.49.x/file-marks.md +++ b/site/content/ytt/docs/v0.49.x/file-marks.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/file-marks] + title: File Marks --- diff --git a/site/content/ytt/docs/v0.49.x/how-it-works.md b/site/content/ytt/docs/v0.49.x/how-it-works.md index f9a4828c7..bdd21c43f 100644 --- a/site/content/ytt/docs/v0.49.x/how-it-works.md +++ b/site/content/ytt/docs/v0.49.x/how-it-works.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/how-it-works] + title: "How it works" --- diff --git a/site/content/ytt/docs/v0.49.x/how-to-export-schema.md b/site/content/ytt/docs/v0.49.x/how-to-export-schema.md index e6fae01d3..1246c3480 100644 --- a/site/content/ytt/docs/v0.49.x/how-to-export-schema.md +++ b/site/content/ytt/docs/v0.49.x/how-to-export-schema.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/how-to-export-schema] + title: Export Schema in OpenAPI Format --- diff --git a/site/content/ytt/docs/v0.49.x/how-to-modularize.md b/site/content/ytt/docs/v0.49.x/how-to-modularize.md index 780c748fa..bdb649dc5 100644 --- a/site/content/ytt/docs/v0.49.x/how-to-modularize.md +++ b/site/content/ytt/docs/v0.49.x/how-to-modularize.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/how-to-modularize] + title: Getting started --- diff --git a/site/content/ytt/docs/v0.49.x/how-to-use-data-values.md b/site/content/ytt/docs/v0.49.x/how-to-use-data-values.md index 99c243ae1..c0962793f 100644 --- a/site/content/ytt/docs/v0.49.x/how-to-use-data-values.md +++ b/site/content/ytt/docs/v0.49.x/how-to-use-data-values.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/how-to-use-data-values] + title: Using Data Values --- diff --git a/site/content/ytt/docs/v0.49.x/how-to-write-schema.md b/site/content/ytt/docs/v0.49.x/how-to-write-schema.md index 2f29ba188..27a8fa7e9 100644 --- a/site/content/ytt/docs/v0.49.x/how-to-write-schema.md +++ b/site/content/ytt/docs/v0.49.x/how-to-write-schema.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/how-to-write-schema] + title: Writing Schema --- diff --git a/site/content/ytt/docs/v0.49.x/how-to-write-validations.md b/site/content/ytt/docs/v0.49.x/how-to-write-validations.md index 2419e7503..6cdfa93e6 100644 --- a/site/content/ytt/docs/v0.49.x/how-to-write-validations.md +++ b/site/content/ytt/docs/v0.49.x/how-to-write-validations.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/how-to-write-validations] + title: Writing Schema Validations --- diff --git a/site/content/ytt/docs/v0.49.x/index-playground-examples.md b/site/content/ytt/docs/v0.49.x/index-playground-examples.md index 7d04cbfca..786b1c60e 100644 --- a/site/content/ytt/docs/v0.49.x/index-playground-examples.md +++ b/site/content/ytt/docs/v0.49.x/index-playground-examples.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/index-playground-examples] + title: Playground Examples Index --- diff --git a/site/content/ytt/docs/v0.49.x/injecting-secrets.md b/site/content/ytt/docs/v0.49.x/injecting-secrets.md index 9e0d095ff..dbd59f8e5 100644 --- a/site/content/ytt/docs/v0.49.x/injecting-secrets.md +++ b/site/content/ytt/docs/v0.49.x/injecting-secrets.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/injecting-secrets] + title: Injecting Secrets --- diff --git a/site/content/ytt/docs/v0.49.x/inputs.md b/site/content/ytt/docs/v0.49.x/inputs.md index f58b6f99d..8fb1de680 100644 --- a/site/content/ytt/docs/v0.49.x/inputs.md +++ b/site/content/ytt/docs/v0.49.x/inputs.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/inputs] + title: Inputs --- diff --git a/site/content/ytt/docs/v0.49.x/install.md b/site/content/ytt/docs/v0.49.x/install.md index 61deecb21..74f88931a 100644 --- a/site/content/ytt/docs/v0.49.x/install.md +++ b/site/content/ytt/docs/v0.49.x/install.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/install] + title: Install --- diff --git a/site/content/ytt/docs/v0.49.x/known-limitations.md b/site/content/ytt/docs/v0.49.x/known-limitations.md index a02a65476..281a8705e 100644 --- a/site/content/ytt/docs/v0.49.x/known-limitations.md +++ b/site/content/ytt/docs/v0.49.x/known-limitations.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/known-limitations] + title: Known Limitations --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-annotation.md b/site/content/ytt/docs/v0.49.x/lang-ref-annotation.md index bfe6a4a12..9647df7a6 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-annotation.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-annotation.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-annotation] + title: Annotations --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-def.md b/site/content/ytt/docs/v0.49.x/lang-ref-def.md index 7b418f365..7229d6e5a 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-def.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-def.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-def] + title: Functions --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-dict.md b/site/content/ytt/docs/v0.49.x/lang-ref-dict.md index e04eaa843..d57eea0ba 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-dict.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-dict.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-dict] + title: Dictionaries --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-for.md b/site/content/ytt/docs/v0.49.x/lang-ref-for.md index d73222ef5..da3e517d4 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-for.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-for.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-for] + title: For loop --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-if.md b/site/content/ytt/docs/v0.49.x/lang-ref-if.md index 914115d3a..0c39cf97b 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-if.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-if.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-if] + title: If Statements --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-list.md b/site/content/ytt/docs/v0.49.x/lang-ref-list.md index 33661f57b..71839f007 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-list.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-list.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-list] + title: Lists --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-load.md b/site/content/ytt/docs/v0.49.x/lang-ref-load.md index a2c764088..9ee9b1b45 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-load.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-load.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-load] + title: Load Statements --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-string.md b/site/content/ytt/docs/v0.49.x/lang-ref-string.md index 4be6bbf5b..ee303f10b 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-string.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-string.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-string] + title: Strings --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-structs.md b/site/content/ytt/docs/v0.49.x/lang-ref-structs.md index aa4ca708c..30f005f74 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-structs.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-structs.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-structs] + title: Structs --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-yaml-fragment.md b/site/content/ytt/docs/v0.49.x/lang-ref-yaml-fragment.md index 547c8a7b8..ec4d02dbf 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-yaml-fragment.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-yaml-fragment.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-yaml-fragment] + title: YAMLFragments --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-assert.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-assert.md index a5f3c3ae3..ea89b95e9 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-assert.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-assert.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-assert] + title: Assert Module --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-library.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-library.md index 3a7a312fb..55043f730 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-library.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-library.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-library] + title: Library Module --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-overlay.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-overlay.md index 7dff6289d..178f008c5 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-overlay.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-overlay.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-overlay] + title: Overlay module --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-schema.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-schema.md index 42160d800..1e10a4849 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-schema.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-schema.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-schema] + title: Data Values Schema Reference --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-struct.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-struct.md index a85bc898e..0c6ca4761 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-struct.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-struct.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-struct] + title: Struct module --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-template.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-template.md index b813ee923..d3cde5b80 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-template.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-template.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-template] + title: Template Module --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-version.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-version.md index 0b70659f2..50fe19940 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt-version.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt-version.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt-version] + title: Version module --- diff --git a/site/content/ytt/docs/v0.49.x/lang-ref-ytt.md b/site/content/ytt/docs/v0.49.x/lang-ref-ytt.md index fb8777d63..5162aff64 100644 --- a/site/content/ytt/docs/v0.49.x/lang-ref-ytt.md +++ b/site/content/ytt/docs/v0.49.x/lang-ref-ytt.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang-ref-ytt] + title: Built-in ytt Library --- diff --git a/site/content/ytt/docs/v0.49.x/lang.md b/site/content/ytt/docs/v0.49.x/lang.md index d46f0a34c..afd0f178d 100644 --- a/site/content/ytt/docs/v0.49.x/lang.md +++ b/site/content/ytt/docs/v0.49.x/lang.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/lang] + title: Language --- diff --git a/site/content/ytt/docs/v0.49.x/outputs.md b/site/content/ytt/docs/v0.49.x/outputs.md index 6c9548312..bf995f80b 100644 --- a/site/content/ytt/docs/v0.49.x/outputs.md +++ b/site/content/ytt/docs/v0.49.x/outputs.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/outputs] + title: Outputs --- diff --git a/site/content/ytt/docs/v0.49.x/schema-validations-cheat-sheet.md b/site/content/ytt/docs/v0.49.x/schema-validations-cheat-sheet.md index e20254858..8ea725539 100644 --- a/site/content/ytt/docs/v0.49.x/schema-validations-cheat-sheet.md +++ b/site/content/ytt/docs/v0.49.x/schema-validations-cheat-sheet.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/schema-validations-cheat-sheet] + title: "Schema Validations Cheat Sheet" --- diff --git a/site/content/ytt/docs/v0.49.x/security.md b/site/content/ytt/docs/v0.49.x/security.md index c8868cf88..8fad7a260 100644 --- a/site/content/ytt/docs/v0.49.x/security.md +++ b/site/content/ytt/docs/v0.49.x/security.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/security] + title: Security --- diff --git a/site/content/ytt/docs/v0.49.x/strict.md b/site/content/ytt/docs/v0.49.x/strict.md index 43a8b7e39..996bf89d6 100644 --- a/site/content/ytt/docs/v0.49.x/strict.md +++ b/site/content/ytt/docs/v0.49.x/strict.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/strict] + title: Strict YAML --- diff --git a/site/content/ytt/docs/v0.49.x/yaml-primer.md b/site/content/ytt/docs/v0.49.x/yaml-primer.md index cf95a8a6d..3c6c8d552 100644 --- a/site/content/ytt/docs/v0.49.x/yaml-primer.md +++ b/site/content/ytt/docs/v0.49.x/yaml-primer.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/yaml-primer] + title: "YAML and Annotations" --- diff --git a/site/content/ytt/docs/v0.49.x/ytt-data-values.md b/site/content/ytt/docs/v0.49.x/ytt-data-values.md index 123d1e62f..8a25f1fce 100644 --- a/site/content/ytt/docs/v0.49.x/ytt-data-values.md +++ b/site/content/ytt/docs/v0.49.x/ytt-data-values.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/ytt-data-values] + title: Data Values --- diff --git a/site/content/ytt/docs/v0.49.x/ytt-overlays.md b/site/content/ytt/docs/v0.49.x/ytt-overlays.md index e3bec98f7..dfbb083b0 100644 --- a/site/content/ytt/docs/v0.49.x/ytt-overlays.md +++ b/site/content/ytt/docs/v0.49.x/ytt-overlays.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/ytt-overlays] + title: Overlays --- diff --git a/site/content/ytt/docs/v0.49.x/ytt-text-templating.md b/site/content/ytt/docs/v0.49.x/ytt-text-templating.md index f2e588508..84b9a0671 100644 --- a/site/content/ytt/docs/v0.49.x/ytt-text-templating.md +++ b/site/content/ytt/docs/v0.49.x/ytt-text-templating.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/ytt-text-templating] + title: Text Templating --- diff --git a/site/content/ytt/docs/v0.49.x/ytt-vs-x.md b/site/content/ytt/docs/v0.49.x/ytt-vs-x.md index fbafe2fcd..2fc329dfb 100644 --- a/site/content/ytt/docs/v0.49.x/ytt-vs-x.md +++ b/site/content/ytt/docs/v0.49.x/ytt-vs-x.md @@ -1,5 +1,5 @@ --- -aliases: [/ytt/docs/latest/ytt-vs-x] + title: ytt vs x --- diff --git a/site/content/ytt/docs/v0.50.x/_index.md b/site/content/ytt/docs/v0.50.x/_index.md new file mode 100644 index 000000000..3cbe6d9a3 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/_index.md @@ -0,0 +1,88 @@ +--- +aliases: [/ytt/docs/latest/] +title: "About ytt" +cascade: + version: v0.50.x + toc: "true" + type: docs + layout: docs +--- + +## Overview + +`ytt` is a command-line tool used to template and patch YAML files. It also provides the means to collect fragments and piles of YAML into modular chunks for easy re-use. + +In practice, these YAML files are [Kubernetes configuration](https://kubernetes.io/docs/concepts/cluster-administration/manage-deployment/), [Concourse Pipeline](https://concourse-ci.org/pipelines.html#schema.pipeline), [Docker Compose](https://github.com/compose-spec/compose-spec/blob/master/spec.md#compose-file), [GitHub Action workflow](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions) files..., really anything that is in YAML format. + +`ytt` is most useful when manually maintaining these files has or will become too much work. + +## Templating + +A plain YAML file is turned into a `ytt` template by adding specially formatted comments, that is: annotating. Through these annotations, one can inject input values, logic like conditionals and looping, and perform transformations on the content. + +Those "input values" are called "Data Values" in `ytt`. Such inputs are included in a separate file. + +For more: +* see it in action in the [load data files](/ytt/#example:example-load-data-values) +example on the playground +* check out the [Using Data Values](how-to-use-data-values.md) guide +- look up reference details of the programming language used in templates in [Language](lang.md). + +## Patching (aka Overlaying) + +`ytt` can also be used to patch YAML files. These edits are called "Overlays" and are themselves written in YAML. + +For more around overlaying... +- see overlaying in action through a progressive set of examples in [the `ytt` Playground](/ytt/#example:example-match-all-docs); +- learn more about Overlays in [ytt Overlays overview](ytt-overlays.md); +- for a reference of all Overlay functionality, see [Overlay module](lang-ref-ytt-overlay.md); +- for a screencast-formatted in-depth introduction to writing and using overlays, watch [Primer on `ytt` Overlays](/blog/primer-on-ytt-overlays/). + +## Modularizing + +`ytt` provides powerful techniques for extracting and reusing chunks of YAML and logic. + +Functions in `ytt` capture either a calculation or fragment of YAML. Functions can be used in the same templates that define them, or — if defined in a module file — loaded into any template. Entire sets of templates, overlays, and library code can be encapsulated in a `ytt` library. + +For more about modular code... +- see live examples in the `ytt` Playground around [functions](ytt/#example:example-function) and [`ytt` libraries](/ytt/#example:example-ytt-library-module); +- read further about [functions](lang-ref-def.md), [YAML Fragments](lang-ref-yaml-fragment.md), and [loading reusable modules and libraries](lang-ref-load.md); +- catch-up on a particularly relevant [discussion about using modules and libraries in `ytt`](https://github.com/carvel-dev/ytt/discussions/392#discussioncomment-766445). + +## Further Reading + +Hopefully, the pointers above helped get you started. If you're looking to go either deeper or broader, here are some resources we can recommend. + +### Documentation + +- [How it Works](how-it-works.md) \ + _a more detailed look of how all these parts fit together._ + +- [ytt vs. X](ytt-vs-x.md) \ + _how `ytt` differs from similar tools._ + +### Articles + +- [ytt: The YAML Templating Tool that simplifies complex configuration management](https://developer.ibm.com/blogs/yaml-templating-tool-to-simplify-complex-configuration-management/) \ + _a complete introduction of `ytt` including motivations and contrasts with other tools._ + +- [Deploying Kubernetes Applications with ytt, kbld, and kapp](/blog/deploying-apps-with-ytt-kbld-kapp) \ + _how `ytt` works well with other Carvel tools._ + +### Presentations and Interviews + +- IBM Developer podcast: [Introducing the YAML Templating Tool (ytt)](https://www.youtube.com/watch?v=KbB5tI_g3bo) \ + _a thorough introduction to `ytt` especially contrasted with Helm._ + +- TGI Kubernetes: [#079: YTT and Kapp](https://www.youtube.com/watch?v=CSglwNTQiYg) \ + _Joe Beda puts both `ytt` and sister tool `kapp` through their paces._ + +- Helm Summit 2019: [ytt: An Alternative to Text Templating of YAML Configuration in Helm](https://www.youtube.com/watch?v=7-PqgpkxC7E) + ([slides](https://github.com/k14s/meetups/blob/develop/ytt-2019-sep-helm-summit.pdf)) \ + _in-depth review of the philosophy behind `ytt` and how it differs from other YAML templating approaches._ + +- Kubecon 2020: [Managing Applications in Production: Helm vs ytt & kapp @ Kubecon 2020](https://www.youtube.com/watch?v=WJw1MDFMVuk) \ + _how `ytt` and `kapp` can provide an alternative way to manage the lifecycle of application on Kubernetes._ + +- Rawkode Live: [Introduction to Carvel](https://www.youtube.com/watch?v=LBCmMTofNxw) \ + _Dmitriy joins David McKay to talk through many tools in the Carvel suite, including `ytt`._ diff --git a/site/content/ytt/docs/v0.50.x/data-values-schema-migration-guide.md b/site/content/ytt/docs/v0.50.x/data-values-schema-migration-guide.md new file mode 100644 index 000000000..f272646f9 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/data-values-schema-migration-guide.md @@ -0,0 +1,207 @@ +--- +aliases: [/ytt/docs/latest/data-values-schema-migration-guide] +title: Schema Migration Guide +toc: "true" +--- + +Schema documents provide a way to declare Data Values with their types, and default values. Without Schema, validating the presence of Data Values requires additional `ytt` configuration containing Starlark assertions. + +- Learn more about [writing Schema](how-to-write-schema.md) +- Read the detailed [Data Vaues Schema Reference](lang-ref-ytt-schema.md) + +## How do I, a configuration author, migrate my `ytt` library to use Schemas? + +To make use of the Schema feature, your `ytt` invocation must first contain files using the [Data Values feature](ytt-data-values.md). Migrating to Schemas involves converting your Data Values files into a Schema file. + +### Single Data Values file + +Starting with a single Data Values file, `values.yml`: +```yaml +#@data/values +--- +key1: myVal +key2: 8080 +``` + +Convert this Data Values file into Schema by changing the top level annotation in the document to say `#@data/values-schema`, and (optional) rename `values.yml` to `values-schema.yml`: +```yaml +#@data/values-schema +--- +key1: myVal +key2: 8080 +``` +Now simply include this Schema file in your `ytt` invocation to receive the benefits of `ytt` Schemas. + + +**Note: If your Data Values file contains arrays (ie. `["example"]`, `- example`), be sure to [provide default values for arrays](#how-do-i-provide-default-values-for-an-array).** + +### Multiple Data Values files +Sometimes, it makes sense to [split Data Values into multiple files](ytt-data-values.md#splitting-data-values-overlays-into-multiple-files). +If this is your situation, there are a few things to note. + +Given a `ytt` configuration with two Data Values files: +```bash +$ tree . +. +├── config.yml +├── values-1.yml +└── values-2.yml +``` + +`values-1.yml`: +```yaml +#@data/values +--- +key1: myVal +key2: 8080 +``` + +`values-2.yml`: +```yaml +#@data/values +--- +key2: 8088 +#@overlay/match missing_ok=True +key3: + host: registry.dev.io + port: 8080 +``` +You can convert each Data Values document into its own Schema document by [following the steps to convert a single Data Values file](#single-data-values-file). + +[Multiple Schemas combine exactly like Data Values, via overlays](lang-ref-ytt-schema.md#multiple-schema-documents): +the first Schema establishes the base set of "data value" declarations, and subsequent Schema files are overlays on top of that base. + +`values-1-schema.yml`: +```yaml +#@data/values-schema +--- +key1: myVal +key2: 8080 +``` + +`values-2-schema.yml`: +```yaml +#@data/values-schema +--- +key2: 8088 +#@overlay/match missing_ok=True +key3: + host: registry.dev.io + port: 8080 +``` + +Now just include these Schema files in your `ytt` invocation instead of the Data Values files. + +### Multiple Data Values files + Private Libraries +If your configuration depends on a ytt library — outlined in the [library module docs](lang-ref-ytt-library.md) — there are a few points to note. + +```bash +$ tree . +. +└── config + ├── config.yml + ├── values-1.yml + ├── values-2.yml + └── _ytt_lib + └── lib + ├── service.yml + └── values.yml +``` +`config.yml`: +```yaml +#@ load("@ytt:data", "data") +#@ load("@ytt:library", "library") +#@ load("@ytt:template", "template") + +#@ lib = library.get("lib").with_data_values(data.values) +--- #@ template.replace(lib.eval()) +``` +Using the same `values-1.yml` and `values-2.yml` files from the [multiple Data Values files Schema migration example above](#multiple-data-values-files). + +Migrating to Schema happens one library at a time. Let's start with the root library, which includes everything at and below the file system level where the `ytt` invocation was called, not including the `_ytt_lib` folder: +```bash +. +└── config + ├── config.yml + ├── values-1.yml + └── values-2.yml +``` +As seen in the [previous example](#multiple-data-values-files), migrating this library to Schemas simply involves converting each `values-1.yml` and `values-2.yml`into a Schema file. + +Now we have migrated the root library to use Schemas, and the `ytt` invocation will succeed as the same as before. Each library can independently opt-in to using Schemas. +```bash +$ tree . +. +└── config + ├── config.yml + ├── values-1-schema.yml + ├── values-2-schema.yml + └── _ytt_lib + └── lib + ├── service.yml + └── values.yml +``` +Migrating a private library to use Schemas involves the same process as the root library. You can narrow the context to just the children of the `_ytt_lib` directory: +```bash +└── _ytt_lib + └── lib + ├── service.yml + └── values.yml +``` +Now simply follow the steps in either of the previous examples to migrate the private library to use Schemas. + +--- + +## How do I provide default values for an array? +Arrays in Schemas are [handled differently](lang-ref-ytt-schema.md#defaults-for-arrays) than other types: +exactly one element is specified in the array, and that value is _only_ used to infer the type of that array's elements — +the default value, by default, is an empty list (i.e. `[]`). + +The example below shows how to define an array in a Schema and then provide default values via the `@schema/default` annotation. + +`values-schema.yml`: +```yaml +#@ def default_conns(): +- host: registry.dev.io + port: 8080 + transport: tcp +#@ end + +#@data/values-schema +--- +#@schema/default default_conns() +key: +- host: "" + port: 0 + transport: "" + insecure_disable_tls_validation: false +``` + +Given that schema, if a template file were to use the `key` data value: + +```yaml +key: #@ data.values.key +``` + +this would output + +```yaml +key: +- host: registry.dev.io + port: 8080 + transport: tcp + insecure_disable_tls_validation: false +``` + + +## How do I mark a section of Data Values as "optional"? +Sometimes your configuration includes a section of Data Values that are not typically used or in some way optional. + +If this the case, consider the guidance in [Writing Schema: Marking a Data Value as Optional](how-to-write-schema.md#marking-a-data-value-as-optional), use the +[@schema/nullalble](lang-ref-ytt-schema.md#schemanullable) annotation to default such a section to `null`. + +## How do I mark a Data Value as containing any kind of YAML? +For those looking to relax the typing that Schema applies to Data Values, the [@schema/type any=True](lang-ref-ytt-schema.md#schematype) annotation +can be used to override the inferred typing on the node it annotates and its children. + +Situations like this are covered in detail in [Writing Schema: Specific Use-Cases](how-to-write-schema.md#specific-use-cases) diff --git a/site/content/ytt/docs/v0.50.x/data-values-vs-overlays.md b/site/content/ytt/docs/v0.50.x/data-values-vs-overlays.md new file mode 100644 index 000000000..78353da0c --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/data-values-vs-overlays.md @@ -0,0 +1,221 @@ +--- +aliases: [/ytt/docs/latest/data-values-vs-overlays] +title: "Data Values vs Overlays" +--- + +## Overview + +As folks get started with `ytt`, a common question that arises is, “when should +I use data values versus overlays?” While these features do address a similar +problem space, we recommend using one feature versus the other depending on the +use case. We will detail our guidance below. + +## Data Values + +[Data values](ytt-data-values.md) provide a way to inject input data into a +template. If you think about a ytt template as a function, then data values are +the varying parameters. The configuration author will expose values that are +likely to change often, such as with every new environment, as data values. +Authors can also set data values to reasonable defaults or leave them empty and +require the consumers to input their own. + +Common use cases: + +1. Set default values (for generic or specific use-cases) +1. Provide visibility to values that could be updated by the consumer +1. Enable simple conditional behavior (such as requiring a value to be set) + +Consider this simple example provided by the configuration author: + +`config.yml` +```yaml +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ data.values.deployment_name or assert.fail("missing deployment_name") + labels: + app: nginx +spec: + replicas: #@ data.values.replicas + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: #@ data.values.nginx_image + ports: + - containerPort: 80 +``` + +`values.yml` +```yaml +#@data/values +--- +deployment_name: "" +replicas: 1 +nginx_image: nginx:1.14.2 +``` + +The configuration consumer can provide their own data values files that override the ones +provided. + +This example portrays how to: + +1. Set default values. The configuration author has set appropriate default + values for `replicas` and `nginx_image`. +1. Provide visibility to values that could be updated by the consumer. + `deployment_name`, `replicas`, and `nginx_image` are all configurable by the + consumer. +1. Enable simple conditional behavior. `deployment_name` must be set by the + consumer, otherwise an error will be returned. + +Pros: + +- Simple (easy to use, easy to understand) + +Cons: + +- Limited in capability (by design) + +--- +## Overlays + +When consumers would like to configure fields beyond what the original +author has exposed as data values, they should turn to +[Overlays](lang-ref-ytt-overlay.md). These documents provide a way to specify +locations within configuration and either add to, remove from, or replace within +that existing configuration. With basic usage, overlays can act as an extension +of data values, but as situations inevitably become more complex, overlays +provide many more capabilities. + +Common use cases: +1. To replace, delete, or append configuration +1. Situation-specific values + +Consider this extension of our previous example: + +`config.yml` +```yaml +#@ load("@ytt:data", "data") +#@ load("@ytt:assert", "assert") +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ data.values.deployment_name or assert.fail("missing deployment_name") + labels: + app: nginx +spec: + replicas: #@ data.values.replicas + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: #@ data.values.nginx_image + ports: + - containerPort: 80 + - name: to-be-removed + image: image:1.2.3 + ports: + - containerPort: 80 +``` + +`values.yml` +```yaml +#@data/values +--- +deployment_name: "" +replicas: 1 +nginx_image: nginx:1.14.2 +``` + +`add-namespace.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +metadata: + #@overlay/match missing_ok=True + namespace: my-namespace +``` + +`remove-container.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +spec: + template: + spec: + containers: + #@overlay/match by="name" + #@overlay/remove + - name: to-be-removed +``` + +`add-container.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +spec: + template: + spec: + containers: + - name: appended-container + image: image:1.2.3 + ports: + - containerPort: #@ data.values.appended_container_port +``` +Prior to v0.32.0 append array items with [`#@overlay/append`](lang-ref-ytt-overlay.md#overlayappend) + +`prod-values.yml` +```yaml +#@data/values +--- +deployment_name: 3 +replicas: 3 +#@overlay/match missing_ok=True +appended_container_port: 8080 +``` + +This example demonstrates a number of overlay capabilities: + +1. Appending configuration via `add-namespace.yml` and `add-container.yml` +1. Removing configuration via `remove-container.yml` +1. Extending and updating data values via `prod-values.yml` + +Use the [ytt playground to play with this +example](/ytt/#gist:https://gist.github.com/aaronshurley/b6868b76e25fcb24aedde42f522734af). + +Pros: +- Has more functionality +- Doesn’t change the source +- Programmatic capabilities (this wasn't demonstrated here, see + [example](lang-ref-ytt-overlay.md#programmatic-access)) + +Cons: +- Added complexity + +# If you want to learn more... + +Check out the [Getting Started +tutorial](/ytt/#example:example-hello-world) on the ytt website +for a detailed introduction to ytt. diff --git a/site/content/ytt/docs/v0.50.x/faq.md b/site/content/ytt/docs/v0.50.x/faq.md new file mode 100644 index 000000000..09f859583 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/faq.md @@ -0,0 +1,189 @@ +--- +aliases: [/ytt/docs/latest/faq] +title: FAQ +--- + +## Data Values + +[Data values doc](ytt-data-values.md) + +## Is it possible to add a new key to my values via the `--data-value` command line argument? +No. As with all data values, those passed through `--data-value` must be overrides, not new values. Instead, overlays are the intended way to provide new keys. +See the [data values vs overlays doc](data-values-vs-overlays.md) for more information. + +## How can I dynamically set or replace map key as a data value in my template? +You can use data.values value as a key in a map by using [Text Templating](ytt-text-templating.md) feature. +That way, you can dynamically set keys in a map using data values. +```yaml +#@yaml/text-templated-strings +(@= data.values.some_key @): some-value +``` +Additionally, see this [playground example](/ytt/#example:example-text-template) which illustrates the use of text templating to set key of a map item. + + +## How do I load json for use as a data value? +An important note here is that json is valid yaml. yaml syntax is a superset of json syntax.\ +ytt can naturally parse json by passing it through `--data-value-yaml`, or json can be loaded by passing the file as a `--data-value-file`. + +Additional resources: [json is valid yaml](https://gist.github.com/pivotaljohn/debe4596df5b4158c7c09f6f1841dd47), [loading by file](https://gist.github.com/pivotaljohn/d3468c3239f79fea7e232751757e779a) + +## How do I check if a key is set? +You can check for the existence of a key by using `hasattr`.\ +For example, to check if struct `foo` has attribute `bar`, use `hasattr(foo, "bar")`. + +## How do I provide a default for a data value when it may not be defined? +When a value may be null, you can use `or` to specify a default. +```yaml +#@ data.values.foo or "bar" +``` + +## How do I error if a data value is not provided? + +ytt library's `assert` package is useful for such situations. It can be used like so: + +```yaml +password: #@ data.values.env.mysql_password if data.values.env.mysql_password else assert.fail("missing env.mysql_password") +``` + +or even more compactly, + +```yaml +password: #@ data.values.env.mysql_password or assert.fail("missing env.mysql_password") +``` + +Note that empty strings are falsy in Starlark. + +--- + +## Overlays + +[Overlays doc](lang-ref-ytt-overlay.md) + +## How do I remove a document subset? + +`#@overlay/remove` in conjunction with `#@overlay/match by=overlay.subset()` annotations are useful for removing a subset of a document. + +Additional resources: [overlay remove docs](lang-ref-ytt-overlay.md#overlayremove), [overlay subset docs](lang-ref-ytt-overlay.md#overlaysubset) + +## How do I add items to an existing array?   + +Using v0.32.0 or later, the default behavior of overlays is to append array items. Simply put your array item in an overlay. + +Prior to v0.32.0, To add an item, either provide the matching annotation (eg. `#@overlay/match by="field_name"`), or use the `#@overlay/append` annotation to add to the end of the list. Note that the append annotation must be applied to each item you want to insert. + +Additional resources: [overlay append docs](lang-ref-ytt-overlay.md#overlayappend), [example gist on playground](/ytt/#gist:https://gist.github.com/pivotaljohn/8c7f48e183158ce12107f576eeab937c), [replace-list gist](/ytt/#gist:https://gist.github.com/pivotaljohn/2b3a9b3367137079195971e1409d539e), [edit-list gist](/ytt/#gist:https://gist.github.com/pivotaljohn/217e8232dc080bb764bfd064ffa9c115) + +## Why am I getting an exception when trying to append to an array? + +A common append issue is incorrectly setting the `#@overlay/match missing_ok=True` annotation on the key which gets replaced by new key-values. Instead, it should be applied to each child (made convenient with the `#@overlay/match-child-defaults missing_ok=True` annotation). See this [illustrative gist](https://gist.github.com/cppforlife/bf42f2d3d23dacf07affcd4150370cb9) for an example. + +## How do I rename a key without changing the value? + +An `#@overlay/replace` annotation with a lambda `via`. For example, to replace the key `bad_name` with `better_name` while retaining the value, you can use: +```yaml +#@overlay/replace via=lambda a,b: {"better_name": a["bad_name"]} +``` +See [this gist](/ytt/#gist:https://gist.github.com/gcheadle-vmware/3c41645a80201caaeefa878e84fff958) for the full example. + +## How do I add or replace a value in a dictionary? + +A `#@ template.replace()` annotation can be used for these purposes. See [this example](/ytt/#example:example-replace). You can also use overlays to edit a dictionary, an example can be found on [this gist playground](/ytt/#gist:https://gist.github.com/gcheadle-vmware/af8aeb3120386e58922c816d76f47ab6). + +## How do I match a field.name that starts with a string? + +```yaml +overlay/match by=lambda a,_: a["field"]["name"].startswith("string") +``` + +## How do I match a struct based on the presence of a key? + +To match a dictionary from a list of dictionaries if the `foo` key is present, you can use +```#@overlay/match by=lambda idx,old,new: "foo" in old, expects="1+"```. + +## How do I modify only part of a multi-line string? + +An `#@overlay/replace` annotation with a lambda `via` function can modify part of a string. See this [modify-string gist](/ytt/#gist:https://gist.github.com/cppforlife/7633c2ed0560e5c8005e05c8448a74d2) for an example. + +## How can I match a regex pattern in the subset matcher? + +The subset matcher does not directly support regex patterns. Instead, a custom matcher can be written. See this [playground gist](/ytt/#gist:https://gist.github.com/ewrenn8/3409e44252f93497a9b447900f3fb5b7) for an example. + +--- + +## When should I include a space in my ytt comment? Does it matter? Is it `#@load` or `#@ load`? `#@overlay/match` or `#@ overlay/match` +The space is subtly meaningful, and directly, load needs a space, while overlay/match does not – but why? + +ytt wraps two concepts in its comment syntax: +1. Annotations on a node +1. Directives for ytt + +Annotations do not have a space, and they refer to a given node in the tree. These comments attach metadata to the annotated node, which can be used during templating. +Some examples of annotations are: +- When inserting a node via an [overlay](#ytt-overlays.md), we would annotate that node with `#@overlay/insert`. +- When we want to mark a document as containing [data values](#ytt-data-values.md), we annotate the document start marker with `#@data/values`. + +Directives, on the other hand, do include a space, and are used to _direct_ ytt to execute the arguments. +Some examples of directives are: +- [Loading](#lang-ref-load.md) a library, we add the `#@ load` directive to the doc, unattached to any particular node. +- To begin and end a [for loop](#lang-ref-for.md) or [conditional](lang-ref-if.md), we use the `#@ for`, `#@ if`, and `#@ end` directives. + +For further exploration, investigate what happens when you move an annotation comment and compare it with moving a directive comment! + +## Why is `ytt` complaining about "Unknown comment syntax"; can't I write standard YAML comments (#)? + +You can, but it is discouraged to avoid tricky errors that can go unchecked. + +The recommended approach is when writing `ytt` templated files, use `ytt` comments: + +``` +#! this is a ytt comment; it's like she-bang! +``` + +If for some reason you need to disable this "linting" check (e.g. you're gradually migrating an existing YAML file to become a `ytt` template and it's onerous to convert all those comments at once), include the `--ignore-unknown-comments` flag. + +For a detailed explanation of how `ytt` detects and processes YAML files, see [File Marks > type detection for YAML files](file-marks.md#type-detection-for-yaml-files). + + +## Why is my anchor reference null despite my anchor's successful template? + +This is a [known limitation](known-limitations.md) of ytt. + +## Can I generate random strings with ytt? +No. A design goal of ytt is determinism, which keeps randomness out of scope. + +If you want to generate secrets, see the [injecting secrets doc](injecting-secrets.md) or the [kubernetes secretgen-controller](https://github.com/carvel-dev/secretgen-controller) + +## Can I load multiple functions without having to name each one? + +Yes! Functions can be stored in a struct which can be imported all together. You can then call individual functions from within that struct. Note that because Starlark does not provide forward references, you must declare the struct that collects the functions to export at the end of the file. + +Storing functions in struct: + +```yaml +#@ load("@ytt:struct", "struct") +#@ mod = struct.make(func1=func1, func2=func2) +``` + +Loading and calling functions in template: + +```yaml +#@ load("helpers.lib.yml", "mod") +something: #@ mod.func1() +``` + +Additional resources: [Load Statement doc](lang-ref-load.md) + +## How do I inject secrets? +See the [injecting secrets doc](injecting-secrets.md). + +## How do I template values within text? +See the [text templating doc](ytt-text-templating.md). Additionally, see this [playground example](/ytt/#example:example-text-template) which illustrates some ways text templating can be done. + +## How can I use files that are symlinks? +ytt takes a secure by default approach to symlinks. It disables use of symlinks to avoid [the risk](security.md#attack-vectors) of malicious template code loading symlinked file contents from sensitive locations. + +If you would like to override this behavior, use `--allow-symlink-destination` flag for allowing symlinks in specific directories or files, or `--dangerous-allow-all-symlink-destinations` to allow all symlinks. + +## What templating language does ytt use? + +ytt uses a fork of [Starlark](https://github.com/bazelbuild/starlark), with a few changes. See the [Language reference](lang.md#Language) for more information. diff --git a/site/content/ytt/docs/v0.50.x/file-marks.md b/site/content/ytt/docs/v0.50.x/file-marks.md new file mode 100644 index 000000000..4f87f2e7f --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/file-marks.md @@ -0,0 +1,138 @@ +--- +aliases: [/ytt/docs/latest/file-marks] +title: File Marks +--- + +## Overview + +ytt allows to control certain metadata about files via `--file-mark` flag. + +```bash +$ ytt ... --file-mark := +``` + +where: +- `path` — location to the file(s) being marked + - exact path (use `--files-inspect` to see paths as seen by ytt) + - path with `*` to match files in a directory + - path with `**/*` to match files and directories recursively +- `mark` — metadata to modify on the file(s) selected by `path` +- `value` — the value for the mark + +Note that this flag can be specified multiple times. + +Example: + +```bash +ytt -f . \ + --file-mark 'alt-example**/*:type=data' \ + --file-mark 'example**/*:type=data' \ + --file-mark 'generated.go.txt:exclusive-for-output=true' \ + --output-files ../../tmp/ +``` + +--- +## Available Marks + +### path + +Changes the relative path. + +``` +--file-mark ':path=' +``` + +Example: + +``` +--file-mark 'generated.go.txt:path=gen.go.txt' +``` + +renames `generated.go.txt` to `gen.go.txt` + +### exclude + +Exclude file from any kind of processing. + +``` +--file-mark ':exclude=true' +``` + +### type + +Change type of file, affecting how `ytt` processes it. + +``` +--file-mark ':type=' +``` + +By default `file-type` is determined based on file extension and — as of v0.32.0 — content. + +`file-type` can be: +- `yaml-template` (default for: `.yml` or `.yaml`) — parsed as a YAML document and evaluated for templating. +- `yaml-plain` — parsed as a simple YAML document (_not_ evaluated for templating). +- `text-template` (default for: `.txt`) — parsed as a text document containing text templating. +- `text-plain` — included in output, as is. +- `starlark` (default for: `.star`) — a Starlark source file (executed) +- `data` (default for all other files) — a text file that can be loaded by [`data.read()`](lang-ref-ytt.md#data) + +Example: + +``` +--file-mark 'config.yml:type=data' +``` + +indicates that `config.yml` is _not_ a `yaml-template`, but is `data`. This file will _not_ be parsed, evaluated, or included in the output, but _can_ be loaded using [`data.read()`](lang-ref-ytt.md#data). + +#### type detection for YAML files +_(as of v0.32.0)_ + +First, `ytt` determines the type of each YAML file: +1. if it has a `.yml` or `.yaml` extension, it's assumed to be a `yaml-template` +2. if it is `yaml-template` but does not contain any ytt templating, it is treated as if it were `yaml-plain` +3. if it is marked with `type=yaml-plain`, it is treated as `yaml-plain` + +and then processes each: +- `yaml-template` files are: + 1. parsed as YAML, + 2. evaluated (i.e. executes `ytt` the template), and + 3. linted (i.e. detects when non-ytt comments are included) + - this catches errors where the `@` is accidentally omitted, + - if the `--ignore-unknown-comments` flag is included, this "linting" is disabled. +- `yaml-plain` files are simply parsed as YAML (with no evaluation or linting). + + +### for-output + +Marks a file to be included in the output. + +Files of [type](#type) `yaml-template`, `yaml-plain`, `text-template`, and `text-plain` are part of the output by default. + +``` +--file-mark ':for-output=true' +``` + +Example +``` +--file-mark 'config.lib.yml:for-output=true' +``` + +By default, `.lib.yml` files are not included in the rendered output (they are loaded +by other templates). With this file mark, `config.lib.yml` _is_ included in the output. + +### exclusive-for-output + +Limits output to only marked files. + +If there is at least one file marked this way, only these files will be used in output. + +``` +--file-mark ':exclusive-for-output=true' +``` + +Example: +``` +--file-mark 'config.lib.yml:exclusive-for-output=true' +``` + +Causes output to only include `config.lib.yml`. diff --git a/site/content/ytt/docs/v0.50.x/how-it-works.md b/site/content/ytt/docs/v0.50.x/how-it-works.md new file mode 100644 index 000000000..f9a4828c7 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/how-it-works.md @@ -0,0 +1,248 @@ +--- +aliases: [/ytt/docs/latest/how-it-works] +title: "How it works" +--- + +## Overview + +Let's get an idea of how `ytt` works by looking at the high-level concepts and flow. + +## The `ytt` Pipeline + +When you invoke `ytt` ... + +```console +$ ytt -f config/ --data-values-file values/ +``` + +... you can think of it as a pipeline in four stages, looking something like this: + +![ytt pipeline overview](/images/ytt/ytt-pipeline-overview.jpg) + + +_(Configuration documents (grey, pale yellow and blue) flow through four pipeline steps (black), into evaluated intermediary documents (bright yellow and blue), and combined ultimately into plain YAML output (green).)_ + +A quick note about files and documents, in general, followed by a detailed description of the elements in this diagram. + +**About YAML files and their documents** + +Each YAML file can have one or more YAML documents in them (separated by `---`). _(from here on, we'll refer to YAML documents as just "documents".)_ + +```yaml +foo: here's the first document (the initial `---` is optional) +--- +bar: this is a separate document +--- +qux: third document's a charm! +``` + +If a given "configuration file" contains one or more `ytt` annotations (i.e. lines that contain `#@`), it's a `ytt` template. + +This is a `ytt` template... +```yaml +--- +foo: 14 +bar: +#@ for/end name in ["Alice", "Bob", "world"]: +- #@ "Hello, " + name +``` +... and, this is plain YAML ... +```yaml +--- +foo: 14 +bar: +- Hello, Alice +- Hello, Bob +- Hello, world +``` + + +### Configuration + +The top-left section of [the diagram](#the-ytt-pipeline) shows the configuration files: the templated configuration and supporting files. + +These files are written and maintained by those who understand how the final result should be shaped. We refer to these folks as Configuration Authors. + +Configuration includes a mixture of these kinds of files: + +- [Data Values Schema Documents](#data-value-schema-documents) +- [Data Values Documents](#data-values-documents) +- [Plain Documents](#plain-documents) +- [Templated Documents](#templated-documents) +- [Overlay Documents](#overlay-documents) + +Now, let's look at each type of document, in turn. + +#### Data Value Schema Documents + +If a document begins with the [`@data/values-schema`](how-to-write-schema.md#starting-a-schema-document) annotation, we call it a "Data Values Schema Document" (the light grey dashed box in the illustration, [above](#the-ytt-pipeline)). + +```yaml +#@data/values-schema +--- +instances: 1 +... +``` + +These files _declare_ variables (Data Values) by setting their name, default value, and type. + +#### Data Values Documents + +When a document starts with the [`@data/values`](how-to-use-data-values.md) annotation, it's called a "Data Values Document" (the light grey dashed box in the illustration, [above](#the-ytt-pipeline)). + +```yaml +#@data/values +--- +instances: 8 +... +``` + +These contain the variables that provide values for templates (explained in more detail, in [Step 2: Evaluate Templates](#step-2-evaluate-templates)). + +#### Plain Documents + +If a document has _no_ `ytt` annotations, we'll call those "Plain Documents" (like the bright yellow item in "Configuration", [above](#the-ytt-pipeline)). +```yaml +--- +notes: +- this will be part of the output +- it does get parsed as YAML, but that's about it +``` +These documents need no processing (outside of being parsed as YAML), and are included as part of the output of the pipeline. + +#### Templated Documents + +If a document _does_ contain templating (i.e. lines containing `#@`) it's known as a "Templated Document" (the _pale_ yellow one, [above](#the-ytt-pipeline)). + ```yaml + #@ load("@ytt:data", "data") + + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: my-app + spec: + replicas: #@ data.values.instances + ... + ``` +These documents are — after being processed — also included as part of the output of the pipeline. + +#### Overlay Documents + +When a document _starts_ with the [`@overlay/match...`](lang-ref-ytt-overlay.md#overlays-as-files) annotation (i.e. above the `---`), it's referred to as an "Overlay Document" (denoted as a pale blue item, [above](#the-ytt-pipeline)): + ```yaml + #@overlay/match by=overlay.all + --- + metadata: + #@overlay/match missing_ok=True + namespace: staging + ... + ``` +These documents describe edits to apply just before generating the output (described in detail, below). + +#### Kinds of Documents in Configuration + +Note that most configuration files can contain any combination of "Plain Documents", "Templated Documents", and "Overlay Documents". + +The exceptions are "Data Values Schema Documents" and "Data Values Documents". These documents must be in their own file, as illustrated [above](#the-ytt-pipeline). + +### Custom Values + +The top-middle section of [the diagram](#the-ytt-pipeline) shows where custom values are injected. + +These are values supplied by those who use `ytt` to generate the final output. We refer to these people as Configuration Consumers. + +They customize the result by supplying their situation-specific settings for Data Values. This can be done as command-line arguments, OS environment variables, and/or plain YAML files. In all cases, these override Data Values that must first be declared in [Data Values Schema Documents](#data-value-schema-documents). + +### Step 1: Calculate Data Values + +Let's explore what happens at each step in the pipeline. + +As the first pipeline step (black box) shows, [above](#the-ytt-pipeline) : + +1. process all the "Data Values Schema" documents (light grey input) — evaluating any templating in them; +2. merge those documents, [in order](lang-ref-ytt-schema.md#multiple-schema-documents), generating a "Data Values Schema" document populated with default values and type information. +3. process all the "Data Values" documents (light grey input) — evaluating any templating in them; +4. merge those documents, [in order](ytt-data-values.md#splitting-data-values-overlays-into-multiple-files). That is, start with the first document and then overlay the second one onto it; then overlay the third document on top of _that_, and so on... +5. finally, override values with the "Custom Values" input, as described in [How to Use Data Values > Configuring Data Values](how-to-use-data-values.md#configuring-data-values). \ + The result of all this is the final set of values that will be available to templates: the dark grey "final Data Values". +6. lastly, if there are any [validations declared in the schema](how-to-write-validations.md), all such rules are evaluated over this final result. + +_(Note the data in-flow arrows into this pipeline step are deliberately ordered, left-to-right, reinforcing the sequence in which values are set: defaults from schema, data values documents, and custom values; last source wins.)_ + + +### Step 2: Evaluate Templates + +Next, evaluate the remaining templates (all the other kinds of "Configuration" documents): +1. "evaluate" means executing all of the [Starlark code](lang.md): loops are run, conditionals decided, expressions evaluated. +1. one notable exception is overlay annotations (i.e. those that start with `@overlay/...`), these are deferred until the next step. +1. a template accesses input variables (i.e. the Data Values calculated in the previous step) via the [`@ytt:data` module](lang-ref-ytt.md#data); + +The result of all this evaluation is a set of YAML documents, configured with the Data Values (shown as "Evaluated Document Set" in the diagram, [above](#the-ytt-pipeline)). + +### Step 3: Apply Overlays + +Note that the "Evaluated Document Set" (see the output from the second step in [the diagram](#the-ytt-pipeline)) contains two groups: +- "Evaluated Documents" — these are the pile of "Plain Documents" and evaluated "Template Documents" (bright yellow) from the previous step. +- "Overlay Documents" — these are the configuration file "Overlay Documents" (bright blue) wherein everything except the `@overlay/...` annotations have been evaluated. + +With these in hand: +1. apply each "Overlay Document" on top of the full set of "Evaluated Documents".\ + You can think of each overlay as like a SQL `update` command: + - the value of it's `by` argument is like a `where` clause that selects over the whole collection of "Evaluated Documents". For example, + ```yaml + #@overlay/match by=overlay.subset({"kind": "Deployment"}), ... + --- + ``` + selects all of the documents which contain a key `"kind"` whose value is `"Deployment"` + - for each of the documents selected, apply the overlay on top of it. This is like a series of `set` clauses, each updating a portion of the document. For example, + ```yaml + #@overlay/match by=overlay.subset({"kind": "Deployment"}), ... + --- + #@overlay/match-child-defaults missing_ok=True + metadata: + labels: + app: frontend + ``` + sets each "Deployment"'s `metadata.labels.app` to be `"frontend"`. +1. repeat that process for each "Overlay Document", [in order](lang-ref-ytt-overlay.md#overlay-order). + +The result is (shown as "Output Document Set" in the diagram, [above](#the-ytt-pipeline)) — the finalized set of YAML documents, in memory. Which leaves one last step... + +### Step 4: Serialize + +This is simply iterating over the "Output Document Set", rendering each YAML Document ("Output Files", [above](#the-ytt-pipeline)). + +The result is sent to standard out (suitable for piping into other tools). If desired, the output can be sent instead to disk using the [`--output...` flags](outputs.md). + +## Further Reading + +We've scratched the surface: an end-to-end flow from pre-processing inputs, processing templates, post-processing overlays, and finally rendering the resulting output. + +To learn more about... +- **Data Values Schema** + - learn about [writing Schema](how-to-write-schema.md) for Data Values + - read-up on the details in the "[Data Values Schema Referce](lang-ref-ytt-schema.md)" material + - work with a complete example from the source: \ + [carvel-dev/ytt/../examples/schema](https://github.com/carvel-dev/ytt/tree/develop/examples/schema) +- **Data Values**... + - poke at a working example in the ytt Playground: [Load Data Values](/ytt/#example:example-load-data-values) example + - read-up on the details in "[Using Data Values](how-to-use-data-values.md)" + - work with a complete example from the source: \ + [carvel-dev/ytt/../examples/data-values](https://github.com/carvel-dev/ytt/tree/develop/examples/data-values) +- **Templates**... + - learn Starlark by example in the first bunch of ["Basics" examples](/ytt/#example:example-plain-yaml). + - read-up on [`ytt`'s built-in libraries](lang-ref-ytt.md) to encode/decode, hash, regex match over data. + - get an helpfully abridged tour of Starlark in "[Language](lang.md)" +- **Overlays**... + - walk through the "Overlays" group of examples: + 1. go to the [ytt Playground](/ytt/#example:example-match-all-docs) + 2. find the "Overlays" group header, click on it to reveal the group _(you can close the "Basics" group by clicking on it)_. + - read an introduction at "[Using Overlays](ytt-overlays.md)" + - watch [Primer on `ytt` Overlays](/blog/primer-on-ytt-overlays/) for a screencast-formatted in-depth introduction to writing and using overlays. + +This overview has meant to provide an end-to-end tour of the core functionality of `ytt`. + +There's more to it: +[modularizing configuration files](how-to-modularize/#extract-functionality-into-custom-library) into reusable Libraries, [text templating](ytt-text-templating.md), and +overriding file processing with [file marks](file-marks.md), and more. Use the left navigation to discover more. diff --git a/site/content/ytt/docs/v0.50.x/how-to-export-schema.md b/site/content/ytt/docs/v0.50.x/how-to-export-schema.md new file mode 100644 index 000000000..e6fae01d3 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/how-to-export-schema.md @@ -0,0 +1,233 @@ +--- +aliases: [/ytt/docs/latest/how-to-export-schema] +title: Export Schema in OpenAPI Format +--- + +The primary use of schema is to declare (and type-check) Data Values. + +`ytt` _also_ supports the ability to export schema definitions in [OpenAPI v3.0.x](https://swagger.io/specification/) format. This is useful for tools that require schema definition in this industry-standard format. For example, `kapp-controller`, requires [Package](/kapp-controller/docs/latest/packaging/#package) 's `spec.valuesSchema.openAPIv3` to contain an [OpenAPI Schema map](https://swagger.io/specification/#schema-object). + +In this way, one can write their schema once and reuse it in situations that require the OpenAPI v3 format. + +## Working Example + +Given a `ytt` schema: + +```yaml +#! schema.yml + +#@data/values-schema +--- +#@schema/title "LoadBalancer type service" +#@schema/desc "Whether to include a LoadBalancer type service and if so, what its IP address is." +load_balancer: + enabled: true + #@schema/nullable + static_ip: "" + +#@schema/title "DNS domains" +#@schema/desc "DNS domains to accept traffic for." +#@schema/default ["apps.example.com", "mobile.example.com"] +app_domains: + #@schema/examples ("Example app domain", "web.myapp.com"), ("","localhost:8080") + - "" + +#@schema/title "Database connections" +#@schema/desc "Connection information for databases used by the system." +#@schema/examples ("Example for local db", [{"name": "default", "adapter": "postgresql", "host": "localhost", "port": 8080}]) +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + +#@schema/title "Additional configuration" +#@schema/desc "Configuration for experimental/optional components; see documentation for more details." +#@schema/examples ("Example of additional config", {"username": "default", "password": "password", "insecureFlag": True}) +#@schema/type any=True +#@schema/deprecated "This data value will be removed in a future release" +additional_config: {} +``` +One can generate the corresponding OpenAPI Document: + +```bash +$ ytt -f schema.yml --data-values-schema-inspect --output openapi-v3 +``` +where: +- `--data-values-schema-inspect` — after all Data Value Schema files have been processed, print the effective schema (instead of rendering any templates). +- `--ouput=openapi-v3` — the schema format to use when rendering is OpenAPI v3.0.x (as opposed to `ytt` formatted schema). + +... which yields: + +```yaml +openapi: 3.0.0 +info: + version: 0.1.0 + title: Schema for data values, generated by ytt +paths: {} +components: + schemas: + dataValues: + type: object + additionalProperties: false + properties: + load_balancer: + title: LoadBalancer type service + type: object + additionalProperties: false + description: Whether to include a LoadBalancer type service and if so, what its IP address is. + properties: + enabled: + type: boolean + default: true + static_ip: + type: string + nullable: true + default: null + app_domains: + title: DNS domains + type: array + description: DNS domains to accept traffic for. + items: + type: string + x-example-description: Example app domain + example: web.myapp.com + default: "" + default: + - apps.example.com + - mobile.example.com + databases: + title: Database connections + type: array + description: Connection information for databases used by the system. + x-example-description: Example for local db + example: + - name: default + adapter: postgresql + host: localhost + port: 8080 + items: + type: object + additionalProperties: false + properties: + name: + type: string + default: "" + adapter: + type: string + default: postgresql + host: + type: string + default: "" + port: + type: integer + default: 5432 + default: [] + additional_config: + title: Additional configuration + nullable: true + deprecated: true + description: Configuration for experimental/optional components; see documentation for more details. + x-example-description: Example of additional config + example: + username: default + password: password + insecureFlag: true + default: {} +``` + +## Exported Properties + +Of the properties declared in the [OpenAPI Schema specification](https://swagger.io/specification/#schema-object), the following are generated. + +### `title` + +(As of v0.39.0+) + +Sets the user-friendly name or title of the node + +- when a data value is annotated `@schema/title`, the value of that title is the value of this property, verbatim; +- otherwise, this property is omitted. + +### `type` + +Names the type of the schema. + +The value of this property depends on the type of data value implied in the `ytt` schema: +- map ==> `object` +- array ==> `array` +- boolean ==> `bool` +- string ==> `string` +- integer ==> `integer` +- float ==> `type: number; format: float` + +### `additionalProperties` + +Indicates whether other keys are allowed in a mapping/object. + +- whenever inspecting a data value that is a map, `ytt` includes this property, setting it to `false`; +- when a data value is annotated `@schema/type any=True`, this property is omitted. + +### `nullable` + +Indicates whether `null` is also allowed. + +- when a data value is annotated `@schema/nullable` or `@schema/type any=True`, this property is included and set to `true`; +- otherwise, this property is omitted. + +### `deprecated` + +(As of v0.39.0+) + +Indicates if a key is deprecated. + +- when a data value is annotated `@schema/deprecated ""`, this property is included and set to `true`; +- requires a string argument intended to explain the deprecation, this string is omitted from the openapi document. +- otherwise, this property is omitted. + +### `description` + +Explains the contents and/or consequences of certain values of the property. + +- when a data value is annotated `@schema/desc`, the value of that description is the value of this property, verbatim; +- otherwise, this property is omitted. + +### `x-example-description` + +(As of v0.39.0+) + +Explains the contents of the `example` property. + +- when a data value is annotated `@schema/examples`, examples are provided via 2-tuples, only the first tuple will be exported to the OpenAPI Document. +- the example description is the first argument of 2-tuple, and must be a string. +- otherwise, this property is omitted. + +### `example` + +(As of v0.39.0+) + +Presents an example value for the data value. + +- when a data value is annotated `@schema/examples`, examples are provided via 2-tuples, only the first tuple will be exported to the OpenAPI Document. +- a 2-tuple contains string description, and an example value: `("first example", exampleYAML())`. +- example values should conform to the type of the same property. +- otherwise, this property is omitted. + +### `default` + +Declares a default value. + +The value of this property: +- is always included; +- is typically the value of the data value in the `ytt` schema; +- if the data value is annotated `@schema/default`, _that_ value is used instead. + +## Known Limitations + +The following are not yet supported in `ytt`: +- `ytt` schema does not yet support declarative validations and thus does not produce such validations in an OpenAPI export. +- there is no means yet to modify the [`info` section of the OpenAPI document](https://swagger.io/specification/#info-object) from within a `--data-values-schema-inspect`. There are defaults supplied, however we recommend that these fields are updated manually. +- inspecting output in `ytt` Schema format is not yet supported. + + + diff --git a/site/content/ytt/docs/v0.50.x/how-to-modularize.md b/site/content/ytt/docs/v0.50.x/how-to-modularize.md new file mode 100644 index 000000000..780c748fa --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/how-to-modularize.md @@ -0,0 +1,477 @@ +--- +aliases: [/ytt/docs/latest/how-to-modularize] +title: Getting started +--- + +## Overview + +Configuration authors looking for examples of how to use functions and variables, [modules](/ytt/docs/develop/lang-ref-load/#files), [data values schema](/ytt/docs/develop/how-to-write-schema/), or a [custom library](/ytt/docs/develop/lang-ref-ytt-library/), will see concrete examples in this guide. Language reference introduces concepts of basic syntax like ytt directives and ytt annotations [definitions](/ytt/docs/develop/faq/#when-should-i-include-a-space-in-my-ytt-comment-does-it-matter-is-it-load-or--load-overlaymatch-or--overlaymatch) (ie:`#@`). See the ytt playground ['getting started'](/ytt/#example:example-hello-world) section for additional examples. + +## Variable and function reuse + +A foundational concept in ytt is using Starlark code to create variables or functions. Inside a YAML file, prefix Starlark code with a ytt annotation `#@ ` (including a space afterwards) to use it inline. + +### Starlark variables +In the code block below there are duplicated values for `name: frontend`, and other values that we may want to modify often. + +```yaml +#! config.yml + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend-deployment + namespace: default + labels: + app.kubernetes.io/version: 0.1.0 + app.kubernetes.io/name: frontend +spec: + selector: + matchLabels: + app: frontend + replicas: 1 + template: + spec: + containers: + - name: frontend + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-service + labels: + app.kubernetes.io/version: 0.1.0 + app.kubernetes.io/name: frontend +spec: + type: ClusterIP + ports: + - port: 80 +``` + +_(This is a ytt comment `#!`, use these instead of YAML comments `#` which are [discouraged](/ytt/docs/develop/faq/#why-is-ytt-complaining-about-unknown-comment-syntax-cant-i-write-standard-yaml-comments-) to ensure that all comments are intentional. Comments will be consumed during execution.)_ + +Using Starlark's Python-like syntax, extract these values into Starlark variables. All the code defined here can be used in the same file. + +```yaml +#! config.yml + +#@ name = "frontend" +#@ namespace = "default" +#@ version = "0.1.0" +#@ replicas = 1 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ name + "-deployment" + namespace: #@ namespace + labels: + app.kubernetes.io/version: #@ version + app.kubernetes.io/name: #@ name +spec: + selector: + matchLabels: + app: #@ name + replicas: #@ replicas + template: + spec: + containers: + - name: #@ name + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: #@ name + "-service" + labels: + app.kubernetes.io/version: #@ version + app.kubernetes.io/name: #@ name +spec: + type: ClusterIP + ports: + - port: 80 +``` +Execute this template by running `ytt -f config.yml`. + +The result is identical to our original template, and now we can be sure all our repeated values will be consistent and easier to modify. + +### Functions +[Functions](lang-ref-def.md) provide a way to extract common code into a separate fragment or code snippet. + +There are two ways to define a function in ytt: as a Starlark function; as a YAML fragment function. + +[Starlark functions](https://github.com/google/starlark-go/blob/master/doc/spec.md#functions) make use of a `return` statement. Because of this they can be great for returning a value that must be transformed in some way. + +[YAML fragment functions](lang-ref-yaml-fragment/#docs) differ in that they are YAML structure wrapped in a Starlark function definition. Everything inside the function will be the return value. They can be great when needing to return nested YAML structure, or key and value pairs. + +Going back to the previous solution, we can see each `labels` key is duplicated YAML, like `app.kubernetes.io/version: #@ version`. There is also some duplicated string manipulation in the `metadata.name` key. +```yaml +#! config.yml + +#@ name = "frontend" +#@ namespace = "default" +#@ version = "0.1.0" +#@ replicas = 1 + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ name + "-deployment" + namespace: #@ namespace + labels: + app.kubernetes.io/version: #@ version + app.kubernetes.io/name: #@ name +spec: + selector: + matchLabels: + app: #@ name + replicas: #@ replicas + template: + spec: + containers: + - name: #@ name + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: #@ name + "-service" + labels: + app.kubernetes.io/version: #@ version + app.kubernetes.io/name: #@ name +spec: + type: ClusterIP + ports: + - port: 80 +``` + +Move the duplicated `labels` keys into a YAML fragment function, and move name formatting into a Starlark function. + +```yaml +#! config.yml + +#@ name = "frontend" +#@ namespace = "default" +#@ version = "0.1.0" +#@ replicas = 1 + +#! Starlark function +#@ def fmt(name, type): +#@ return "{}-{}".format(name, type) +#@ end + +#! YAML fragment function +#@ def labels(name, version): +app.kubernetes.io/version: #@ version +app.kubernetes.io/name: #@ name +#@ end + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ fmt(name, "deployment") + namespace: #@ namespace + labels: #@ labels(name, version) +spec: + selector: + matchLabels: + app: #@ name + replicas: #@ replicas + template: + spec: + containers: + - name: #@ name + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: #@ fmt(name, "service") + labels: #@ labels(name, version) +spec: + type: ClusterIP + ports: + - port: 80 +``` +Execute this template by running `ytt -f config.yml`. +Again, the result is identical to our original template, and we can be sure all our repeated sections of YAML will be consistent. + +--- +## Externalize a value with data values schema + +Use [Data values schema](how-to-write-schema.md) to externalize a configuration value. When externalizing a value, you can also set a default value for it, and provide implicit type validation. Data values schema are used by configuration authors to declare a data value as an input to templates by naming it in a schema file. It can then be used in templates, and configuration consumers can modify it by providing a separate [Data Value](how-to-use-data-values.md) file. + +Building on the previous solution, `name`, `namespace`, `version`, and `replicas` are values we want as data values input, so that we can easily change them for different applications or environments. + +```yaml +#! schema.yml +#@data/values-schema +--- +name: "frontend" #! ensures that any value for 'frontend' must be a string +namespace: "default" #! ensures that any value for 'default' must be a string +version: "0.1.0" #! ensures that any value for 'version' must be a string +replicas: 1 #! ensures that any value for 'replicas' must be a int +``` +```yaml +#! config.yml + +#@ load("@ytt:data", "data") + +#@ def fmt(name, type): +#@ return "{}-{}".format(name, type) +#@ end + +#@ def labels(name, version): +app.kubernetes.io/version: #@ version +app.kubernetes.io/name: #@ name +#@ end + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ fmt(data.values.name, "deployment") + namespace: #@ data.values.namespace + labels: #@ labels(data.values.name, data.values.version) +spec: + selector: + matchLabels: + app: #@ data.values.name + replicas: #@ data.values.replicas + template: + spec: + containers: + - name: #@ data.values.name + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: #@ fmt(data.values.name, "service") + labels: #@ labels(data.values.name, data.values.version) +spec: + type: ClusterIP + ports: + - port: 80 +``` + +Execute ytt via `ytt -f config.yml -f schema.yml`. +The result is identical to our original template. + +## Extract code into modules + +Modules contain code in a file that can be imported and used in templates. + +Starlark modules have the `.star` extension. + +YAML modules have the `.lib.yml` extension. + +These two files are imported identically. The main difference is that Starlark modules contain only Starlark code, and YAML modules are YAML structures with Starlark code contained in `#@` annotations, just like the code we have seen thus far. + +### Starlark module +Following the last solution, move the `fmt()` function to a separate Starlark file. +```python +#! format.star + +def fmt(name, type): + return "{}-{}".format(name, type) +end +``` +Import the module by loading it `#@ load("format.star", "fmt")`. + +### YAML module +Move the `labels()` function to a separate YAML file. + +```yaml +#! labels.lib.yml + +#@ def labels(name, version): +app.kubernetes.io/version: #@ version +app.kubernetes.io/name: #@ name +#@ end +``` +Import the module by loading it `#@ load("labels.lib.yml", "labels")`. + +The load function takes a module file path, and secondly the name of the function or variable to export from the module. For multiple symbols, use a comma separated list of strings. If your module has many symbols that are usually all exported together, consider putting them in a [struct](faq/#can-i-load-multiple-functions-without-having-to-name-each-one), and load that struct. + +```yaml +#! config.yml + +#@ load("@ytt:data", "data") +#@ load("labels.lib.yml", "labels") +#@ load("format.star", "fmt") + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: #@ fmt(data.values.name, "deployment") + namespace: #@ data.values.namespace + labels: #@ labels(data.values.name, data.values.version) +spec: + selector: + matchLabels: + app: #@ data.values.name + replicas: #@ data.values.replicas + template: + spec: + containers: + - name: #@ data.values.name + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: #@ fmt(data.values.name, "service") + labels: #@ labels(data.values.name, data.values.version) +spec: + type: ClusterIP + ports: + - port: 80 +``` + +```yaml +#@data/values-schema +--- +name: "frontend" +namespace: "default" +version: "0.1.0" +replicas: 1 +``` +```shell +$ tree . +. +├── config.yml +├── schema.yml +├── format.star +└── labels.lib.yml +``` + +Execute ytt via `ytt -f .` to include all files in this directory. + +## Extract functionality into custom library +You can extract a whole set of input files (i.e. templates, overlays, data values, etc.) into a "Library" in the `_ytt_lib/` folder. A library can be thought of as a separate self-contained ytt invocation. Libraries are _not_ automatically included in `ytt` output. They must be programmatically imported using the [`library` module](/ytt/docs/develop/lang-ref-ytt-library/), configured, evaluated, and inserted into a template that is part of the output. + +### Uses of a custom library +Libraries are helpful when +* importing 3rd party configuration into one combined piece of configuration. Such as from a division of responsibility or shared configuration. + * For example, having a library that provides helpful functions that needs to be used across multiple teams. Authors may use a tool to ensure these stay in sync. +* A template is needed by two distinct applications like frontend and backend. +* There is a need to update one application with an evaluated value from the other. [Playground example here](/ytt/#example:example-ytt-library-module). + +All the previous section's files have moved to `_ytt_lib/resources`: + +```shell +config/ +$ tree . +├── _ytt_lib/ +│ └── resources/ +│ ├── config.yml +│ ├── schema.yml +│ ├── labels.lib.yml +│ └── format.star +├── config.yml +└── values.yml +``` +In this example we want the resources from the library for a frontend application, and also for a backend application. We will use this library to create both of these. + +Focusing on only the two top level files, `config.yml`, and `values.yml`, we import the custom library, provide it data values for a frontend and separately for a backend, and use `#@ template.replace()` to insert it inline so it shows in the output. + +```yaml +#! config.yml +#@ load("@ytt:data", "data") +#@ load("@ytt:library", "library") +#@ load("@ytt:template", "template") + +#@ resources_lib = library.get("resources") +#@ backend = resources_lib.with_data_values(data.values.backend) +#@ frontend = resources_lib.with_data_values(data.values.frontend) + +--- #@ template.replace(backend.eval()) +--- #@ template.replace(frontend.eval()) +``` +Provide the values that we pass to the library. +```yaml +#@data/values-schema +--- +frontend: + name: "frontend" + namespace: "dev" + replicas: 1 + version: "0.5.0" +backend: + name: "backend" + namespace: "dev" + replicas: 1 + version: "0.2.0" +``` + +Run ytt with `.` to include all files in this directory. + +```shell +$ ytt -f . +``` +The result is the similar to our original template with resources configured for two different applications. +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: backend-deployment + namespace: dev + labels: + app.kubernetes.io/version: 0.2.0 + app.kubernetes.io/name: backend +spec: + selector: + matchLabels: + app: backend + replicas: 1 + template: + spec: + containers: + - name: backend + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: backend-service + labels: + app.kubernetes.io/version: 0.2.0 + app.kubernetes.io/name: backend +spec: + type: ClusterIP + ports: + - port: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: frontend-deployment + namespace: dev + labels: + app.kubernetes.io/version: 0.5.0 + app.kubernetes.io/name: frontend +spec: + selector: + matchLabels: + app: frontend + replicas: 1 + template: + spec: + containers: + - name: frontend + image: index.docker.io/k14s/image@sha256:6ab29951e0207fde6760f6db227f218f20e875f45b22e8ca0ee06c0c8cab32cd +--- +apiVersion: v1 +kind: Service +metadata: + name: frontend-service + labels: + app.kubernetes.io/version: 0.5.0 + app.kubernetes.io/name: frontend +spec: + type: ClusterIP + ports: + - port: 80 +``` + diff --git a/site/content/ytt/docs/v0.50.x/how-to-use-data-values.md b/site/content/ytt/docs/v0.50.x/how-to-use-data-values.md new file mode 100644 index 000000000..99c243ae1 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/how-to-use-data-values.md @@ -0,0 +1,141 @@ +--- +aliases: [/ytt/docs/latest/how-to-use-data-values] +title: Using Data Values +--- + +## Overview + +A Configuration Author introduces variables in `ytt` (i.e. to externalize configuration values) by: +1. declaring them as "Data Values" by naming it in a schema file and then, +2. referencing them in templates. + +Configuration Consumers then set values for those variables in any combination of: +- one or more of the `--data-value...` flag(s) and/or +- Data Values Overlay(s) through the `--file` flag + +This guide illustrates how to declare and configure data values. + +_(for a higher-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +### Declaring Data Values + +In `ytt`, before a Data Value is used, it is declared. This is typically done in a schema file. + +For example: + +`schema.yml` +```yaml +#@data/values-schema +--- +name: monitor +ingress: + virtual_host_fqdn: "monitor.system.example" + service_port: 80 + enable_tls: false +``` + +declares five Data Values: +- `name` contains a string; the default name is "monitor". + - `ingress` is a map that contains three map items: `virtual_host_fqdn`, `service_port`, and `enable_tls`. +- `ingress.virtual_host_fqdn` is a string; by default, the fully-qualified host name is the value given. +- `ingress.service_port` is an integer; by default, the service is listening on the standard HTTP port. +- `ingress.enable_tls` is a boolean; by default, transport layer security is off. + +_(see the [How To Write Schema](how-to-write-schema.md) guide, for details.)_ + + +### Referencing Data Values + +Those Data Values can then be referred to in template(s): + +`config.yml` +```yaml +#@ load("@ytt:data", "data") +--- +name: #@ data.values.name +spec: + virtualhost: #@ data.values.ingress.virtual_host_fqdn + services: + - port: #@ data.values.ingress.service_port + #@ if/end data.values.ingress.enable_tls: + - port: 443 +``` +where: +- `load("@ytt:data", "data")` imports the the `data` struct from the `@ytt:data` module +- `data.values` contains all of the declared data values +- `#@ if/end` only includes the annotated array item if the data value `ingress.enable_tls` is true. + +Using the defaults given in the schema, `ytt` produces: +```console +$ ytt -f schema.yml -f config.yml +name: monitor +spec: + virtualhost: monitor.system.example + services: + - port: 80 +``` + +_(For details on using the Data module, refer to [`@ytt:data`](lang-ref-ytt.md#data).)_ + +### Configuring Data Values + +Those Data Values can be configured by a Consumer: + +This is done, typically, via a Data Values File: + +`values.yml` +```yaml +--- +name: observer +ingress: + virtual_host_fqdn: "observer.system.example" + enable_tls: true +``` + +which is a plain YAML file (i.e. _cannot_ contain any `ytt` templating). This file is specified through the `--data-values-file` flag. + +Using the example files from above, `ytt` produces this output: + +```console +$ ytt -f schema.yml -f config.yml --data-values-file values.yml +name: observer +spec: + virtualhost: observer.system.example + services: + - port: 80 + - port: 443 +``` + +Supplied Data Values are automatically checked against the schema. If any value is of the wrong type, `ytt` reports the discrepancies and stops processing. + +_(For details on how to configure Data Values, consult the [Data Values](ytt-data-values.md) reference.)_ + + +## Resources + +Documentation: +- [How To Write Schema](how-to-write-schema.md) guide — step-by-step writing schema in `ytt`. +- [Data Values Reference](ytt-data-values.md) — details of how Data Values are specified in all scenarios. +- [Data Values Schema Reference](lang-ref-ytt-schema.md) — the anatomy of a `ytt` Schema file all elements within. +- [Schema Migration Guide](data-values-schema-migration-guide.md) — migrating existing `ytt` code from pre-Schema versions. + +Examples: +- Declaring and using Data Values in schema: \ + https://github.com/carvel-dev/ytt/tree/develop/examples/schema +- Setting a value for an _array_ in schema: \ + https://github.com/carvel-dev/ytt/tree/develop/examples/schema-arrays +- Using most of the `--data-value...` flags:\ + https://github.com/carvel-dev/ytt/tree/develop/examples/data-values/ +- Marking a data value as "required":\ + https://github.com/carvel-dev/ytt/tree/develop/examples/data-values-required/ +- Maintaining per-environment data value overrides:\ + https://github.com/carvel-dev/ytt/tree/develop/examples/data-values-multiple-envs +- Wrapping an upstream set of templates to expose a simplified set of data values:\ + https://github.com/carvel-dev/ytt/tree/develop/examples/data-values-wrap-library +- Using a directory full of YAML files for data values input:\ + https://github.com/carvel-dev/ytt/tree/develop/examples/data-values-directory + +Blog Articles: +- [Parameterizing Project Configuration with ytt](https://carvel.dev/blog/parameterizing-project-config-with-ytt/), by Garrett Cheadle +- [Deploying to multiple environments with ytt and kapp](https://carvel.dev/blog/multi-env-deployment-ytt-kapp/), by Yash Sethiya + diff --git a/site/content/ytt/docs/v0.50.x/how-to-write-schema.md b/site/content/ytt/docs/v0.50.x/how-to-write-schema.md new file mode 100644 index 000000000..2f29ba188 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/how-to-write-schema.md @@ -0,0 +1,283 @@ +--- +aliases: [/ytt/docs/latest/how-to-write-schema] +title: Writing Schema +--- + +## Overview + +In `ytt`, before a Data Value can be used in a template, it must be declared. This is typically done via Data Values Schema. + +This guide shows how to write such schema. + +_(For a broader overview of Data Values, see [Using Data Values](how-to-use-data-values.md))._ + +--- +## Starting a Schema Document + +One writes Data Values Schema in a YAML document annotated with `#@data/values-schema`: + +```yaml +#@data/values-schema +--- +#! Schema contents +``` + +Files containing Schema documents are included via the `--file`/`-f` flag. + +```bash +$ ytt ... -f schema.yml ... +``` + +The contents of such a document _are_ the Data Values being declared. + +## Declaring Data Values + +Each item in the schema document declares a Data Value. + +A Data Value declaration has three (3) parts: a **name**, a **default value**, and a **type**. + +For example, +```yaml +#@data/values-schema +--- +system_domain: "" +``` + +declares the Data Value, with the name `system_domain` to be of type **string** with the default value of an **empty string** (i.e. `""`). + +--- +## Implying Types + +In `ytt` Schema types of Data Values are _inferred_ from the values given. So, when one writes schema, they are _implying_ the type of each Data Value through the default value they give. + +For example: +```yaml +#@data/values-schema +--- +system_domain: "" + +load_balancer: + enabled: true + static_ip: "" + +app_domains: +- "" + +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + user: admin + secretRef: + name: "" +``` + +effectively declares the following data values: +- `system_domain` — a string +- `load_balancer` — a map containing two items: + - `enabled` — a boolean + - `static_ip` — a string +- `app_domains` — an array of strings (and only strings) +- `databases` — an array where each element is a map. Each map has exactly six items: + - `name` — a string + - `adapter` — a string + - `host` — a string + - `port` — an integer + - `user` — a string + - `secretRef` — a map having exactly one item: + - `name` — a string + +_(see [Data Values Schema Reference: Types](lang-ref-ytt-schema.md#types) for details.)_ + +--- +## Setting Default Values + +In `ytt` Schema, the default value for a Data Value is almost always simply the value specified. + +From the example, [above](#implying-types), the corresponding Data Values have the following defaults: +- `system_domain` is empty string (i.e. `[]`), +- `load_balancer.enabled` is `true`, and +- `load_balancer.static_ip` initializes to empty string. + +For details on how to set individual default values, see [Data Values Schema Reference: Default Values](lang-ref-ytt-schema.md#default-values). + +**Special Case: Arrays** + +There is one exception: arrays. As described in [Data Values Schema Reference: Defaults for Arrays](lang-ref-ytt-schema.md#defaults-for-arrays), the default value for arrays, by default, is an empty list (i.e. `[]`). That said, when an item is added to the array, _that item's_ value is defaulted as defined in the schema. + +In the example, [above](#implying-types), the definition of `databases` is an array. Each item in _that_ array is a map with six keys including `adapter`, `port`, etc. + +`database` starts out as an empty list. Then, as each item added to it, they will be defaulted with the values given in the schema. + +Continuing with our example schema, [above](#implying-types), if a Data Values overlay gives an actual value for the array: + +```yaml +#@data/values +--- +databases: +- name: core + host: localhost +``` + +That item will be filled-in with defaults: + +```yaml +name: core +adapter: postgresql +host: localhost +port: 5432 +user: admin +secretRef: + name: "" +``` + +In order to override the default of the array, _itself_, within schema see [Setting a Default Value for Arrays](#setting-a-default-value-for-an-array). + +## Constraining values with Validations + +To learn about writing schema validations, please refer to [How to Write Schema Validations](how-to-write-validations.md)\ +To get started quickly - [Schema validations Cheat Sheet](schema-validations-cheat-sheet.md) + +## Specific Use-Cases + +A few less common, but real-world scenarios: + +- [setting a default value for arrays](#setting-a-default-value-for-an-array) +- [marking a Data Value as optional](#marking-a-data-value-as-optional) +- [allowing multiple types of maps or arrays](#allowing-multiple-types-of-maps-or-arrays) +- [declaring "pass-through" Data Values](#declaring-pass-through-data-values) + +### Setting a Default Value for an Array + +As explained in [Data Values Schema Reference: Defaults for Arrays](lang-ref-ytt-schema.md#defaults-for-arrays), unlike all other types, the default value for an array is an empty list (i.e. `[]`). + +In some cases, it is useful to provide a non-empty default value for an array. To do so, one uses the `@schema/default` annotation. + +For example, with this schema: + +```yaml +#@data/values-schema +--- +#@schema/default ["apps.cf-apps.io", "mobile.cf-apps.io"] +app_domains: +- "" +``` + +The default value for `app_domains` will be `["apps.cf-apps.io", "mobile.cf-apps.io"]`. + +See also: [@schema/default](lang-ref-ytt-schema.md#schemadefault). + +### Marking a Data Value as Optional + +Sometimes, it can be useful to define a section of Data Values as optional. This typically means that templates that rely on those values conditionally include output that use the contained value. + +For example, the following template: +```yaml +#@ load("@ytt:data", "data") +--- +... +spec: + #@ if data.values.load_balancer: + loadBalancerIP: #@ data.values.load_balancer.static_ip + #@ end +... +``` +will only include `spec.loadBalancerIP` if a value is provided for the `load_balancer` Data Value. + +One notes this in Schema using the `@schema/nullable` annotation: + +```yaml +#@data/values-schema +--- +#@schema/nullable +load_balancer: + static_ip: "" +``` + +which indicates that `load_balancer` is `null`, by default. However, if a value _is_ provided for `load_balancer`, it must be the `static_ip` and have a value that is a **string**. + +For more details see [Data Values Schema Reference: `@schema/nullable`](lang-ref-ytt-schema.md#schemanullable). + +### Marking a Data Value as Required + +In `ytt`, a data value can be marked as "Required" using schema validations. +Please refer to +["Required" Data Values](/ytt/docs/develop/how-to-write-validations/#required-data-values) + +### Allowing Multiple Types of Maps or Arrays + +In rare cases, a given Data Value needs allow more than one type. + +Currently, `ytt` Schema does not explicitly support specifying more than one type for a Data Value. + +In the meantime, one can mark such Data Values as having [`any` Type](lang-ref-ytt-schema.md#any-type): + +```yaml +#@schema/type any=True +int_or_string: "" +``` +so that: +- `int_or_string` is, by default, an empty string +- it can accept an **integer** or a **string** ... or any other type, +- and the value of `int_or_string` and its children is not checked by schema. + +If it is critical to ensure that the type of `int_or_string` to be _only_ an integer or string, one can include a validating library that does so explicitly: + +```python +load("@ytt:assert", "assert") +load("@ytt:data", "data") + +if type(data.value.int_or_string) not in ("int", "string"): + assert.fail("'int_or_string' must be either an integer or a string.") +end +``` + +### Declaring "Pass-through" Data Values + +In certain cases, one designs a Data Value to carry a chunk of YAML whose exact type or shape is unimportant to templates. In these situations, it is undesirable for that chunk of YAML to be type-checked. + +One can effectively disable schema type checking and defaulting by marking the Data Value as of type "any": + +```yaml +#@data/values-schema +--- +honeycomb: + enabled: false + api_key: so124me14v4al1i5da5p5i180key + #@schema/type any=True + optional_config: null +``` + +Here, `additional_config` can contain any valid YAML. + +For example: + +```yaml +#@data/values +--- +honeycomb: + optional_config: + default_series: + id: 1001 + description: Administrative Actions +``` + +In a template, the Data Value can be referenced and its contents will be inserted: + +```yaml +#@ load("@ytt:data", "data") +--- +#@ if/end data.values.honeycomb.enabled: +honeycomb: + config: + api_key: #@ data.values.honeycomb.api_key + #@ if/end data.values.honeycomb.optional_config: + optional: #@ data.values.honeycomb.optional_config +``` + + +--- +## Next Steps + +Once you've declared your Data Values, they can be referenced in `ytt` templates as described in [Using Data Values > Referencing Data Values](how-to-use-data-values.md#referencing-data-values) diff --git a/site/content/ytt/docs/v0.50.x/how-to-write-validations.md b/site/content/ytt/docs/v0.50.x/how-to-write-validations.md new file mode 100644 index 000000000..2419e7503 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/how-to-write-validations.md @@ -0,0 +1,573 @@ +--- +aliases: [/ytt/docs/latest/how-to-write-validations] +title: Writing Schema Validations +--- + +## Overview + +_(Looking for a quick start? see the [Validations Cheat Sheet](schema-validations-cheat-sheet.md))_ + +A Configuration Author can constraint their users' Data Value inputs via `ytt` Validations. + +One might do this for a number of reasons: +- **catch configuration errors early** — help the user of the `ytt` library from wasting time _discovering_ errors in their configuration when they use it... by catching and _reporting_ those errors right away + - e.g. in Kubernetes, instead of sifting through statuses and logs troubleshooting a failed deploy, present the user with an error message at configuration time — _before_ the deployment begins. +- **avoid impractical configuration** — guide them away from setting values that won't work in practice (e.g. too many `replicas:` of a Kubernetes Deployment, when the system won't actually scale that large); +- **make a Data Values "required"** — force the user to supply values for Data Values that you — as the author — can't possibly know (e.g. credentials, connection info to services, etc.). + +This guide explains how to do all that with Validations. + +## What Validations Look Like + +A validation is an annotation on a Data Value in a schema file: + +```yaml +#@data/values-schema +--- +dex: + #@schema/validation min_len=1 + namespace: "" +``` + +Here: +- the Data Value `dex.namespace` is a string +- to be valid, that string must be set to a value at least one character long. + +One can specify multiple "rules": + +```yaml +#@data/values-schema +--- +dex: + #@schema/validation min_len=1, max_len=63 + namespace: "" +``` + +Here, `dex.namespace` must ultimately be: +- at least 1 character, _and_ +- no more than 63 characters in length. + +And one can declare validations for _each_ Data Value; `ytt` will validate _all_ Data Values, together. + +```yaml +#@data/values-schema +--- +dex: + #@schema/validation min_len=1, max_len=63 + namespace: "" + #@schema/validation min_len=1 + username: "" +``` + +Here: +- additionally, `dex.username` is also a string; and to be valid must be at least one character in length. + +> _There can only be one `@schema/validation` annotation on a Data Value: all rules required to define validity must be combined into that one annotation._ + +Finally, one can write their own custom rules, as well: + +```yaml +#@data/values-schema +--- +dex: + #@schema/validation ("not 'default'", lambda v: v != "default"), min_len=1, max_len=63 + namespace: "" +``` + +Here: +- For `dex.namespace` to be valid it must: + - _not_ be the string "default", _and_ + - be at least one character long, _and_ + - be no more than 63 characters in length. + +_(For a list of the most common rules, see [Validations Cheat Sheet](schema-validations-cheat-sheet.md))_ + +_(For details on the use and shape of individual rules, see [About Rules](#about-rules), below)_ + +## How Validations Work + +At a high level, validations fit into the flow like so: + +1. In schema, the Author declares validations on Data Values (described [above](#what-a-validations-look-like)) +2. The Consumer configures data values (described in [How To Use Data Values](how-to-use-data-values.md#configuring-data-values)) +3. All of those values are merged into a single set (the first step of the [ytt Pipeline](how-it-works.md#step-1-calculate-data-values)) +4. All validations are run on that final Data Values; if any fail, those are collected as "violations" +5. If there were violations, processing stops and those are reported; \ + ... otherwise, processing continues normally. + +For example, given this schema: + +```yaml +#@data/values-schema +--- +dex: + #@schema/validation min_len=1, max_len=63 + namespace: "" + #@schema/validation min_len=1 + username: "" +``` + +If the Consumer supplies no data values, then the final Data Values are the defaults from the schema. +When the validations are run, instead of rendering templates, `ytt` reports the violations: + +```console +$ ytt -f schema.yaml +ytt: Error: Validating final data values: + dex.namespace + from: schema.yaml:5 + - must be: length >= 1 (by: schema.yaml:4) + found: length = 0 + + dex.username + from: schema.yaml:7 + - must be: length >= 1 (by: schema.yaml:6) + found: length = 0 +``` + +And if the Consumer supplies data values: + +```yaml +dex: + namespace: the-longest-namespace-you-ever-did-see-in-fact-probably-too-long + username: alice +``` + +Only _after_ those are merged with the default values are validations run: + +```console +$ ytt -f schema.yaml --data-values-file values.yaml +ytt: Error: Validating final data values: + dex.namespace + from: alues.yaml:2 + - must be: length <= 63 (by: schema.yaml:4) + found: length = 64 +``` + +Until, finally, all Data Values have valid values: + +```console +$ ytt -f schema.yaml --data-values-file values.yaml --data-value dex.namespace=ident-system --data-values-inspect +dex: + namespace: ident-system + username: alice +``` + +Next, we cover typical situations where validations are useful... + +## Common Use Cases + +There are a variety of ways you can put validations to use: + +- ["Required" Data Values](#required-data-values) — ensure the user supplies their own value +- [Enumerations](#enumerations) — limit the value to a finite, specific set. +- [Mutually Exclusive Sections](#mutually-exclusive-sections) — when there are multiple way to configure a feature, but the Consumer should only choose one. +- [Conditional Validations](#conditional-validations) — trigger validations only in certain situations. + +### "Required" Data Values + +Sometimes, there are configuration values that you — as the Author — either can't possibly know (e.g. IP addresses, domain names) or do not want to default (e.g. passwords, tokens, certificates, other credentials). Instead, you want to force the Consumer to supply these values. + +> ℹ️ _The way to mark a Data Value as "required" is by declaring a validation rule that is not satisfied by that Data Value's default._ + +There are three general tactics: + +- ideally, [using natural constraints](#using-natural-constraints) +- otherwise, [using the empty/zero value](#using-the-emptyzero-value) +- if all else fails, [mark as 'nullable' _and_ 'not_null'](#mark-as-nullable-and-not_null) + +#### Using Natural Constraints + +The most concise (and maintainable) way to make a data value "required" is to set a default outside of its natural constraints. + +For example, if `port:`, means any "registered" (ports 1024 - 49151) or "dynamic" (49152 - 65535) port, then + +```yaml +#@data/values-schema +--- +#@schema/validation min=1024 +port: 0 +``` + +out of the box, the Consumer receives this message: + +```console +$ ytt -f schema.yaml + port + from: schema.yaml:4 + - must be: a value >= 1024 (by: schema.yaml:3) + found: value < 1024 +``` + +Where there are not "natural" limits, one might be able to use the zero or empty value... + +#### Using the empty/zero value + +For strings, an empty value is often not valid. One can specifically require a non-zero length: + +```yaml +#@schema/validation min_len=1 +username: "" +``` + +For integers and floating-point values, non-positive numbers are often not valid. One can require a non-negative number + +```yaml +#@schema/validation min=1 +replicas: 0 +``` + +For array values, note that [the default value is always an empty list](how-to-write-schema.md#setting-a-default-value-for-an-array). One can require that the array not be empty: + +```yaml +#@data/values-schema +--- +dex: + oauth2: + #@schema/validation min_len=1 + responseTypes: + - "" +``` +Here, +- `dex.oauth2.responseTypes` is an array of strings. +- by default no response types are configured. +- however, the rule requires that _at least one_ be specified. + +#### Mark as 'nullable' and 'not_null' + +In some cases, there simply is no invalid value and/or there is no zero value (e.g. maps). + +What's left is to specify no value at all (i.e. `null`) and then require a non-null value. + +```yaml +#@data/values-schema +--- +#@schema/nullable +#@schema/validation not_null=True +tlsCertificate: + tls.crt: "" + tls.key: "" + ca.crt: "" +``` +Here: +- `tlsCertificate:` is a map, containing three items. +- `@schema/nullable` changes `tlsCertificate:` in two ways (details at [`@schema/nullable`](lang-ref-ytt-schema.md#schemanullable)) + - now, `tlsCertificate` can be set to `null` + - and, `tlsCertificate` is `null` by default. +- the `not_null=` rule requires that `tlsCertificate:` _not_ be `null` + +out of the box, the Consumer receives this message: + +```console +$ ytt -f schema.yaml +ytt: Error: Validating final data values: + tlsCertificate + from: schema.yaml:5 + - must be: not null (by: schema.yaml:4) + found: value is null +``` + +### Enumerations + +Some values must be from a discrete and specific set. + +```yaml +#@data/values-schema +--- +#@schema/validation one_of=["aws", "azure", "vsphere"] +provider: vsphere +``` + +### Conditional Validations + +Sometimes, a Data Value should be validated only when some _other_ configuration has been set. + +In `ytt` Validations, this is achieved through the `when=` keyword. + +For example: + +```yaml +#@data/values-schema +--- +#@schema/validation ("at least 1 instance", lambda v: v["instances"] >= 1), when=lambda v: v["enabled"] +service: + enabled: true + instances: 1 +``` +Here: +- if `service.enabled` is false, the validation is _not_ run; +- when `service.enabled` is true, `service.instances` is required to be non-negative. + +_(For more details, see [Reference for `@schema/validation`](lang-ref-ytt-schema.md#schemavalidation).)_ + +#### Making Validations Dependent on Other Data Values + +In some situations, a Data Values's final value is only relevant (i.e. worth validating) if some _other_ data value has a specific setting. +For these situations, the `@schema/validation ... when=` can accept an optional second parameter. + +For example, the previous example could be rewritten as: + +```yaml +#@data/values-schema +--- +service: + enabled: true + #@schema/validation min=1, when=lambda _, ctx: ctx.parent["enabled"] + instances: 6 +``` +where: +- the `when=` now has a function value that accepts two (2) arguments; the second of which is named `ctx`. + - see [Reference for `@schema/validation`](lang-ref-ytt-schema.md#schemavalidation) for details of the value assigned to `ctx`. +- `instances` will only be validated if `enabled` is `true` + +### Mutually Exclusive Sections + +One pattern found in configuration files is the "mutually exclusive" structure. + +This is typically done with a discriminator field: + +```yaml +--- +dex: + config: + type: "oidc" + oidc: + CLIENT_ID: null #! required if oidc enabled + CLIENT_SECRET: null #! required if oidc enabled + issuer: null #! is required if oidc enabled + ldap: + host: null #! is required if ldap enabed + bindDN: null + bindPW: null +``` +Here: +- there are two kinds of identity systems one could configure: OIDC _or_ LDAP. +- which one being used is named in `dex.config.type`; in this case, OIDC. +- both structures are present and `null` values are used. + +Essential is that the Consumer can configure _either_ OIDC _or_ LDAP _but not both._ + +There are at least a couple of approaches possible: +- [Using `one_not_null=`](#using-one_not_null) to enforce that only one section can be populated. +- [Using a Discriminator as the Condition](#using-a-discriminator-as-the-condition) to trigger validations only on the currently selected section. + +#### Using `one_not_null=` + +With `ytt` the Author can more clearly enforce this structure and validate only the active configuration: + +```yaml +#@data/values-schema +--- +dex: + #@schema/validation one_not_null=["oidc", "ldap"] + config: + #@schema/nullable + oidc: + CLIENT_ID: "" + CLIENT_SECRET: "" + issuer: "" + #@schema/nullable + ldap: + host: "" + bindDN: "" + bindPW: "" +``` +Here: +- each option (i.e. `oidc:` and `ldap:`) are made "optional" by marking them as `@schema/nullable` + - `@schema/nullable` makes a Data Value _able_ to be `null` _and_ sets it to `null`, by default. (for details [`@schema/nullable`](lang-ref-ytt-schema.md#schemanullable)) +- however, `config:` requires that _exactly one (1)_ of the listed keys contain a **not-null** value. + +By default, then, neither `oidc:` nor `ldap:` are configured: + +```console +$ ytt -f schema.yaml --inspect-data-values --dangerous-data-values-disable-validation +dex: + config: + oidc: null + ldap: null +``` + +When validations run, the Consumer is prompted to configure one: + +```console +$ ytt -f schema.yaml --data-values-inspect +ytt: Error: Validating final data values: + dex.config + from: schema.yaml:5 + - must be: exactly one of ["oidc", "ldap"] to be not null (by: schema.yaml:4) + found: all values are null +``` + +Once a value _is_ provided for one or the other, the configuration becomes valid: + +```console +$ ytt -f schema.yaml --data-values-inspect --data-value dex.config.oidc.CLIENT_ID=admin +dex: + config: + oidc: + CLIENT_ID: admin + CLIENT_SECRET: "" + issuer: "" + ldap: null +``` + +#### Using a Discriminator as the Condition + +In some cases, it may be desirable to keep the discriminator. + +Reworking the example from above... + +```yaml +#@data/values-schema +--- +dex: + config: + #@schema/validation one_of=["oidc", "ldap"] + type: "oidc" + oidc: + #@schema/validation min_len=1, when=lambda _, ctx: ctx.root["dex"]["config"]["type"] == "oidc" + CLIENT_ID: "" + #@schema/validation min_len=1, when=lambda _, ctx: ctx.root["dex"]["config"]["type"] == "oidc" + CLIENT_SECRET: "" + #@schema/validation min_len=1, when=lambda _, ctx: ctx.root["dex"]["config"]["type"] == "oidc" + issuer: "" + ldap: + #@schema/validation min_len=1, when=lambda _, ctx: ctx.root["dex"]["config"]["type"] == "ldap" + host: "" + #@schema/validation min_len=1, when=lambda _, ctx: ctx.root["dex"]["config"]["type"] == "ldap" + bindDN: "" + #@schema/validation min_len=1, when=lambda _, ctx: ctx.root["dex"]["config"]["type"] == "ldap" + bindPW: "" +``` +where: +- `dex.config.type` can only be _either_ "oidc" or "ldap" +- `dex.config.oidc` values will only be validated if `type` is "oidc" +- `dex.config.ldap` values, likewise, will only be validated if `type` is "ldap" + +_(See also [Making Validations Dependent on Other Data Values](#making-validations-dependent-on-other-data-values).)_ + +## About Rules + +A validation is made up of one or more rules. + +A rule comes in one of two forms: +- a [named rule](#using-named-rules) — a set of pre-built commonly used rules +- a [custom rule](#writing-your-own-custom-rules) — a rule an Author writes for a specific purpose + +### Using "Named" Rules + +`ytt` comes with a library of built-in rules known as "named" rules. + +These rules are primarily used as a keyword on the `@schema/validation` annotation; refer to the [Data Values Schema Reference](lang-ref-ytt-schema.md#schemavalidation) for the current complete list. Most of the examples we see use named rules. + +For example: +```yaml +#@data/values-schema +--- +#@schema/validation one_of=["INFO", "WARN", "ERROR", "FATAL"] +logLevel: INFO +``` +Here: +- `logLevel` is a string, defaulting to "INFO" +- a valid `logLevel` must be one of the four values given. + +Authors are encouraged to use named rules whenever possible: +- there's no code to maintain: these rules are [unit-tested](https://github.com/carvel-dev/ytt/tree/develop/pkg/validations/filetests) +- they more succinctly document the constraints, making the schema easier to read/maintain +- when rules are included in [OpenAPI v3 schema exports](how-to-export-schema.md), these are the first batch of such rules likely to be included. + + +### Writing Custom Rules + +The ["Named" rules](#using-named-rules) will not cover _all_ possible validation cases. One might opt to write a custom rule for a number of reasons: +- the desired constraint can't be expressed through a named rule +- the description supplied by a named rule is inadequate + + +- [Complex Custom Rules](#complex-custom-rules) +- [About `null` values](#about-null-values) + +A validation rule has two parts: +- a description of a valid value; +- a function that implements that definition in Starlark code. + +For example: +```yaml +#@data/values-schema +--- +#@schema/validation ("a multiple of 1024", lambda v: v % 1024 == 0) +quota: 1023 +``` +Here: +- the rule is a two-value "[tuple](https://github.com/google/starlark-go/blob/master/doc/spec.md#tuples)" +- the first value is a string; + - it describes what a valid value looks like; + - this string is used in violation messages (see below) to help the user provide a valid input. +- the second value is a function (here's a [lambda](https://github.com/google/starlark-go/blob/master/doc/spec.md#lambda-expressions) expression); + - the function will be passed one (1) argument: the value being validated. + - the function must return a boolean value (either `True` or `False`) _or_ `fail()` with a message describing the failure. + +Has the initial result: + +```console +$ ytt -f schema.yaml +ytt: Error: Validating final data values: + quota + from: schema.yaml:4 + - must be: a multiple of 1024 (by: schema.yaml:3) +``` + +And quietly reports nothing when the value _is_ valid. + +#### Complex Custom Rules + +Occasionally, a validation rule requires pre-processing of a value or requires multiple checks. In these cases, a lambda expression is often not enough: a function needs to be written. + +To keep the schema itself readable/maintainable, Authors will typically extract these functions to a separate file: + +`rules.star` +```python +def one_registry_if_pvc_is_filesystem(val): + if val["persistence"]["imageChartStorage"]["type"] == "filesystem" and \ + val["persistence"]["persistentVolumeClaim"]["registry"]["accessMode"] == "ReadWriteOnce": + return val["registry"]["replicas"] == 1 \ + or fail("{} replicas are configured".format(val["registry"]["replicas"])) + end +end +``` + +`schema.yaml` +```yaml +#@ load("rules.star", "one_registry_if_pvc_is_filesystem") + +#@data/values-schema +#@schema/validation ("exactly one (1) registry replica if Helm Charts are stored on the filesystem.", one_registry_if_pvc_is_filesystem) +--- +registry: + replicas: 2 +persistence: + imageChartStorage: + type: "filesystem" + persistentVolumeClaim: + registry: + accessMode: "ReadWriteOnce" +``` + +Here: +- the `@schema/validation` annotates the _document_ because the validation applies across two top-level keys (`registry` and `persistence`); +- the assertion itself is defined in `rules.star`, so as to not clutter the schema; it is _loaded_ into `schema.yaml` +- in `rules.star`, the parameter `val` is expected to receive the value of the document: + - that value is a [YAML Fragment](lang-ref-yaml-fragment.md) containing a map with those two top-level keys (and their contents, recursively) + - and so, contained items are accessed through bracket notation. + +_(See also using a `struct` to export multiple functions through a single `load()` in [Load Statements > Usage](lang-ref-load.md#usage))_ + +#### About `null` values + +`ytt` attempts to gracefully handle `null` values in validations: +- when a Data Value is marked `@schema/nullable`, and the value remains `null`, validations are skipped automatically. +- if the same Data Value has the `not_null=True` rule, _that_ rule is run. +- the `not_null=` rule, if present, is checked _first_. + +The upshot of these policies are: +- no other rules need handling the `null` value. + diff --git a/site/content/ytt/docs/v0.50.x/index-playground-examples.md b/site/content/ytt/docs/v0.50.x/index-playground-examples.md new file mode 100644 index 000000000..7d04cbfca --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/index-playground-examples.md @@ -0,0 +1,160 @@ +--- +aliases: [/ytt/docs/latest/index-playground-examples] +title: Playground Examples Index +--- + +## Basics + +### How Plain YAML is parsed + +https://carvel.dev/ytt/#example:example-plain-yaml + +### Datatypes in Starlark are encoded in YAML + +https://carvel.dev/ytt/#example:example-datatypes + +### Local variables within a template + +https://carvel.dev/ytt/#example:example-variable + +### Surrounding YAML with an if statement + +https://carvel.dev/ytt/#example:example-if + +### Looping or Generating YAML with a for loop + +https://carvel.dev/ytt/#example:example-for + +### Making a mini-template or extracting a chunk of YAML using a function + +https://carvel.dev/ytt/#example:example-function + +### Including helper functions from another file using load() + +https://carvel.dev/ytt/#example:example-load + +### Encoding or hashing a string + +https://carvel.dev/ytt/#example:example-load-ytt-library-module + +### Including a function from a library + +https://carvel.dev/ytt/#example:example-load-custom-library-module + +### Asserting on a particular value or condition + +https://carvel.dev/ytt/#example:example-assert + +### Declaring and using Data Values + +https://carvel.dev/ytt/#example:example-load-data-values + +### Loading and including data from a text or CSV file + +https://carvel.dev/ytt/#example:example-load-data-files + +### Templating strings and key names using text templating + +https://carvel.dev/ytt/#example:example-text-template + +### Inserting a YAML fragment or a value using template.replace() + +https://carvel.dev/ytt/#example:example-replace + +### Modifying or editing a YAML fragment with an overlay + +https://carvel.dev/ytt/#example:example-overlay + +### Writing an overlay that applies over all the other documents + +https://carvel.dev/ytt/#example:example-overlay-files + +### Splitting data values into multiple files + +https://carvel.dev/ytt/#example:example-multiple-data-values + +### Grouping and using multiple templates as a set using libraries + +https://carvel.dev/ytt/#example:example-ytt-library-module + +### Simple Kubernetes example with a Pod and a Service + +https://carvel.dev/ytt/#example:example-k8s-ingress-single + +### Kubernetes example producing multiple applications from data values + +https://carvel.dev/ytt/#example:example-k8s-ingress-multiple + +### Helm Chart like output from ytt + +https://carvel.dev/ytt/#example:example-k8s-helm-ish + +### A complete working example of many ytt features + +https://carvel.dev/ytt/#example:example-demo + +## Overlays + +### Overlay matching on all documents + +https://carvel.dev/ytt/#example:example-match-all-docs + +### Overlay matching on specific documents + +https://carvel.dev/ytt/#example:example-match-subset-docs + +### Overlay matching on specific documents using a YAML Fragment + +https://carvel.dev/ytt/#example:example-match-subset-by-fragment + +### Overlay matching on a specific document by index + +https://carvel.dev/ytt/#example:example-match-by-index + +### Overlay matching on an array item by a specific field or key + +https://carvel.dev/ytt/#example:example-match-by-key + +### Overlay that inserts an array item at a specific spot + +https://carvel.dev/ytt/#example:example-insert-array-item + +### Overlay that replaces an array item + +https://carvel.dev/ytt/#example:example-replace-array-item + +### Overlay that removes or deletes an array item + +https://carvel.dev/ytt/#example:example-remove-array-item + +### Overlay that edits or modifies an array item in place + +https://carvel.dev/ytt/#example:example-edit-array-item + +### Overlay that appends an array item (inserts at the end) + +https://carvel.dev/ytt/#example:example-append-array-item + +### Overlay that completely replaces the contents of an array + +https://carvel.dev/ytt/#example:example-replace-array + +### Overlay that adds a new map item or field + +https://carvel.dev/ytt/#example:example-add-map-item + +### Overlay that edits the value of a map item or field + +https://carvel.dev/ytt/#example:example-edit-map-value + +### Overlay that removes or deletes a map item or field + +https://carvel.dev/ytt/#example:example-remove-map-item + +### Overlay that renames the key or field of a map item + +https://carvel.dev/ytt/#example:example-rename-key-in-map + +### Overlay that add or appends a map into an array item + +https://carvel.dev/ytt/#example:example-append-map-to-array diff --git a/site/content/ytt/docs/v0.50.x/injecting-secrets.md b/site/content/ytt/docs/v0.50.x/injecting-secrets.md new file mode 100644 index 000000000..9e0d095ff --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/injecting-secrets.md @@ -0,0 +1,160 @@ +--- +aliases: [/ytt/docs/latest/injecting-secrets] +title: Injecting Secrets +--- + +## Overview + +**This document is work in progress.** + +Unlike most configuration, many organizations disallow storing of plain secret values next to other code/configuration. + +This document: +- presents several common approaches used to make secrets available to your templates +- does _not_ cover injection of secrets directly into an application at runtime (overall may be the best approach) +- does _not_ recommend one approach over another (though it does state pros and cons) +- does _not_ talk about where resulting templates are forwarded + +One common question that's asked is why not to extend ytt to allow it to shell out to other programs or why not include builtin library that can fetch secrets from outside (e.g. imagine having builtin `vault` function). We are _not_ planning to support either of these features because we want to maintain strong sandbox boundary between templating language and ytt's external environment. Our expectation is that if ytt user wants to provide specific data to templates, it should be injected into ytt rather than templates (or ytt itself) fetching it. We believe this is a good security posture and as a side effect makes templates more portable (some users may store secrets in Vault and some other in Lastpass). + +- [Via command line args](#via-command-line-args) +- [Via environment variables](#via-environment-variables) +- [Via secret references, before templating](#via-secret-references-before-templating) +- [Via encrypted secrets, before templating](#via-encrypted-secrets-before-templating) +- [Via secret references, after templating](#via-secret-references-after-templating) +- [Via encrypted resources, later decrypted inside the k8s cluster](#via-encrypted-resources-later-decrypted-inside-the-k8s-cluster) + +There might be other ways to inject secrets into ytt that are not listed here. + +--- +## Via command line args + +```bash +ytt -f . --data-value username=user --data-value password=pass +``` + +Cons: + +- depending on a computing environment secret values are visible in a process tree (as part of full command line) **which typically is considered unwanted** + +--- +## Via environment variables + +```bash +export CFG_secrets__username=user +export CFG_secrets__password=pass +ytt -f . --data-values-env CFG +``` + +Cons: + +- depending on a computing environment secret values are visible in process metadata (e.g. typically accessible to root user under `/proc//environ`) + +--- +## Via secret references, before templating + +Given following two files: + +`config.yml`: + +```yaml +#@ load("@ytt:data", "data") + +users: +- username: #@ data.values.username + password: #@ data.values.password +``` + +`secrets`: + +```yaml +username: (( vault "secret/my/credentials/admin:username" )) +password: (( vault "secret/my/credentials/admin:password" )) +``` + +(Note that `secrets` file does not have `yaml` extension to make ytt exclude it from template result. Alternatively `--file-mark` flag can be used to exclude that file.) + +One can compose ytt with external tooling that knows how to resolve referenced values. For example, [spruce](https://starkandwayne.com/blog/simple-secure-credentials-into-yaml-with-vault-and-spruce/) can resolve secret references and then forward that information onto ytt: + +```bash +echo -e "#@data/values\n---\n$(spruce merge ./secrets)" | ytt -f . -f - +``` + +Pros: + +- able to commit mapping of data values to secrets in your preferred secret storage (in this example Vault) +- actual secret values are available to templates, hence, can be inserted in the middle of a string or base64 encoded (as sometimes required by k8s) + +--- +## Via encrypted secrets, before templating + +Similar to above "Via secret references, before templating" approach, instead of storing secret references in `secrets` file one can store encrypted secret values in `secrets` next to other configuration. Various tools like [sops](https://github.com/mozilla/sops) make encrypting configuration files fairly easy: + +```bash +echo -e "#@data/values\n---\n$(sops -d ./secrets)" | ytt -f . -f - +``` + +Pros: + +- provides a way to version (backup, etc) secrets next to other configuration +- actual secret values are available to templates, hence, can be inserted in the middle of a string or base64 encoded (as sometimes required by k8s) + +Cons: + +- depending on organization requirements even encrypted secrets may not be allowed to be stored next to other configuration + +--- +## Via secret references, after templating + +Given following two files: + +`config.yml`: + +```yaml +#@ load("@ytt:data", "data") + +users: +- username: #@ data.values.username + password: #@ data.values.password +``` + +`values.yml`: + +```yaml +#@data/values +--- +username: (( vault "secret/my/credentials/admin:username" )) +password: (( vault "secret/my/credentials/admin:password" )) +``` + +One can compose ytt with external tooling that knows how to resolve referenced values. Such tools typically look for a particular patterned map or array items. One such tool is [spruce](https://starkandwayne.com/blog/simple-secure-credentials-into-yaml-with-vault-and-spruce/): + +```bash +ytt -f . | spruce merge - +``` + +Pros: + +- provides a way to commit mapping of data values and their associated secret locations (e.g. `username` comes from `vault` under `secret/my/credentials/admin:username`) + +Cons: + +- secret resolution tools may not be able to find secret references inside strings (e.g. `http://(( ... )):(( ... ))@website`) +- secret resolution tools will not understand if secret reference was encoded (for example with `base64` encoding as wanted sometimes by k8s) + +--- +## Via encrypted resources, later decrypted inside the k8s cluster + +This approach is Kubernetes specific. + +[Bitnami's `sealed-secrets` project](https://github.com/bitnami-labs/sealed-secrets) provides a way to encrypt secrets hence allowing to commit them next to your other configuration. With this approach, secret values are not accessible to templates directly; however, they can be referenced by k8s resources (for example via `secretRef` in `Pod` spec). At the time of the deploy of such resources, sealed secrets controller will create appropriate `Secret` objects with decrypted secret values. + +Pros: + +- provides a way to version (backup, etc) secrets next to other configuration +- no way to access decrypted secret values in templates directly (though, this may not be necessary in many cases) + +Cons: + +- Kubernetes specific +- depending on organization requirements even encrypted secrets may not be allowed to be stored next to other configuration diff --git a/site/content/ytt/docs/v0.50.x/inputs.md b/site/content/ytt/docs/v0.50.x/inputs.md new file mode 100644 index 000000000..f58b6f99d --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/inputs.md @@ -0,0 +1,94 @@ +--- +aliases: [/ytt/docs/latest/inputs] +title: Inputs +--- + +ytt supports different input sources: + +- Files & Directories + - Those are provided via the `-f`/`--file` flag + + - ytt uses the file's name for its internal representation + + If you have a tree like + ```terminal + $ tree . + . + ├── dir1 + │   └── some.yaml + └── dir2 + └── sub + ├── another.yaml + └── someother.yaml + + 4 directories, 3 files + ``` + and you call `ytt --file dir1/some.yaml --file dir2/ ...` then ytt loads + - `dir1/some.yaml` as `some.yaml` + - `dir2/sub/another.yaml` as `sub/another.yaml` + - `dir2/sub/someother.yaml` as `sub/someother.yaml` + + - ytt uses a file's extension to determine its type, e.g. a extension like + `yaml` flags that file as "yaml-template"; you can read more about that in + [File Marks](file-marks/) + + - You can change a file's name, location, and also "type" by explicitly + setting the file's name to be used by ytt, e.g. `ytt --file + a/different/file.foo=dir1/some.yaml`, which would mean that ytt + + - loads that file as `a/different/file.foo` + - would not consider it as "yaml-template"/"yaml-plain", but as "data", because of its extension + + Note: this only works for files, not for directories + + - Explicitly setting file's names can be especially useful when consuming + files where you have no control over their name, like process substitutions: + + Running `ytt --file <(echo 'some: yaml')` (on Linux) would have the shell + produce a file like `/dev/fd/63` and pass that on to ytt. This file, based + on it's name "63", would not be considered yaml and thus interpreted as + "data". To change that, you need to run `ytt --file subst.yaml=<(echo 'some: + yaml')` to have ytt treat it as yaml. + + - ytt can also consume stdin by using `-`, like: `ytt --file -` + + Note: When using `-`, ytt automatically treats data on stdin as yaml, as it + will use stdin as `stdin.yaml`, thus having an extension which flags it as + "yaml-template". If you use some other means to consume stdin, e.g. `ytt + --file /dev/stdin`, this does not happen and ytt treats stdin as a file + `stdin` and thus as "data", because it has no extension marking it + differently. You can still set a different file name explicitly, e.g. with + `ytt --file my-stdin.yaml=/dev/stdin`. + + - ytt can also consume files via http/s, e.g. `ytt --file + https://raw.githubusercontent.com/carvel-dev/ytt/develop/.golangci.yml` + + - ytt can also consume symlinks, however if a symlink's target is not a file + you have already included into the set of files ytt should consider + (`--file ...`), ytt will not allow that and print an error. You can + explicitly allow additional symlink targets via the + `--allow-symlink-destination ...` flag. + + - To debug / inspect which files ytt considers and how it handles those, the + flags `--files-inspect` & `--debug` can be helpful + + +- Data Values & Data Values Schemas + + You can read about how to define data-values schemas and how to consume and + set Data Values and Schemas here: + + - [Write Schema](how-to-write-schema/) + - [Data Values Schema](lang-ref-ytt-schema/) + - [Use Data Values](how-to-use-data-values/) + - [Data Values](ytt-data-values/) + + Generally you can provide Data Values + - as strings + - on the command line via the flags `--data-value`/`-v` + - from the environment via the flag `--data-values-env` + - from files via the flag `--data-value-file` + - as structured data / yaml + - on the command line via the flag `--data-value-yaml` + - from the environment via the flag `--data-values-env-yaml` + - from files via the flag `--data-values-file` diff --git a/site/content/ytt/docs/v0.50.x/install.md b/site/content/ytt/docs/v0.50.x/install.md new file mode 100644 index 000000000..61deecb21 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/install.md @@ -0,0 +1,66 @@ +--- +aliases: [/ytt/docs/latest/install] +title: Install +--- + +## Via script (macOS or Linux) + +(Note that `install.sh` script installs other Carvel tools as well.) + +Install binaries into specific directory: + +```bash +$ mkdir local-bin/ +$ curl -L https://carvel.dev/install.sh | K14SIO_INSTALL_BIN_DIR=local-bin bash + +$ export PATH=$PWD/local-bin/:$PATH +$ ytt version +``` + +Or system wide: + +```bash +$ wget -O- https://carvel.dev/install.sh > install.sh + +# Inspect install.sh before running... +$ sudo bash install.sh +$ ytt version +``` + +## Via Homebrew (macOS or Linux) + +Based on [github.com/carvel-dev/homebrew](https://github.com/carvel-dev/homebrew). + +```bash +$ brew tap carvel-dev/carvel +$ brew install ytt +$ ytt version +``` + +## Specific version from a GitHub release + +To download, click on one of the assets in a [chosen GitHub release](https://github.com/carvel-dev/ytt/releases), for example for 'ytt-darwin-amd64'. + +```bash +# **Compare binary checksum** against what's specified in the release notes +# (if checksums do not match, binary was not successfully downloaded) +$ shasum -a 256 ~/Downloads/ytt-darwin-amd64 +08b25d21675fdc77d4281c9bb74b5b36710cc091f30552830604459512f5744c /Users/pivotal/Downloads/ytt-darwin-amd64 + +# Move binary next to your other executables +$ mv ~/Downloads/ytt-darwin-amd64 /usr/local/bin/ytt + +# Make binary executable +$ chmod +x /usr/local/bin/ytt + +# Check its version +$ ytt version +``` +## Shell Completion + +The `ytt completion` command generates an autocompletion script for the specified shell. + +See `ytt completion --help` for information and instructions. + +For detailed instructions on enabling shell completion, specify the type of shell in the help command. For example: +`ytt completion zsh --help`. diff --git a/site/content/ytt/docs/v0.50.x/known-limitations.md b/site/content/ytt/docs/v0.50.x/known-limitations.md new file mode 100644 index 000000000..a02a65476 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/known-limitations.md @@ -0,0 +1,12 @@ +--- +aliases: [/ytt/docs/latest/known-limitations] +title: Known Limitations +--- + +- YAML anchors and templating directive for the same YAML node are not supported. + + ```yaml + first: &content #@ 123 + second: *content + ``` + `second` key-value pair will _not_ contain 123 since YAML anchors are resolved before ytt evaluates templating directives. diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-annotation.md b/site/content/ytt/docs/v0.50.x/lang-ref-annotation.md new file mode 100644 index 000000000..bfe6a4a12 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-annotation.md @@ -0,0 +1,54 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-annotation] +title: Annotations +--- + +## Format + +``` +@ann1-name [ann-arg1, ann-arg2, ..., keyword-ann-arg1=val1] +``` + +- content between brackets is optional. + +- annotation names are typically namespaced, for example, `overlay/merge` is an annotation within an `overlay` namespace. Annotation namespaces are there for general organization, they are not associated with loaded packages (from `load` keyword). + +- annotation arguments (positional and keyword) is just plain code + +## Shared templating annotations + +- `@template/code [code]` or just `@ [code]` (on its own line) represents plain code line + +```yaml +#@ a = calculate(100) +key: value +``` + +- `@template/value [code]` or just `@ [code]` (at the end of line) represents a value associated structure + +```yaml +key: #@ value +array: +- #@ value +``` + +## YAML templating annotations + +- `@yaml/map-key-override` (no args) + - necessary to indicate that map key is being intentionally overriden + +- `@yaml/text-templated-strings` (no args) + - necessary to indicate that node contents (including map key and map value) should be text templated (ie `(@` indicates start of text templating) (see [text templating](ytt-text-templating.md) for more details.) + +## Text templating annotations + +- `@text/trim-left` trims space to the left of code node +- `@text/trim-right` trims space to the right of code node + +## Data module annotations + +- `@data/values` (no args) specifies values accessible via `data.values` from `@ytt:data` package (see [ytt @data/values](ytt-data-values.md) for more details) + +## Overlay module annotations + +See [Overlay module](lang-ref-ytt-overlay.md) for list of annotations. diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-def.md b/site/content/ytt/docs/v0.50.x/lang-ref-def.md new file mode 100644 index 000000000..7b418f365 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-def.md @@ -0,0 +1,73 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-def] +title: Functions +--- + +Refer to [Starlark function specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#functions) for details about various types of function arguments. Note that ytt's Starlark use requires functions to be closed with an `end`. + +- function definition within YAML + +Labels returns map with two keys: test1, and test2: + +```yaml +#@ def my_labels(): +test1: 123 +test2: 124 +#@ end +``` + +Above is _almost_ equivalent to (differnce is that return type in one case is a YAML fragment and in another it's a dict): + +```yaml +#@ def my_labels(): +#@ return {"test1": 123, "test2": 124} +#@ end +``` + +- function definition within Starlark (.star files) + +```python +def my_labels(): + return {"test1": 123, "test2": 124} +end +``` + +- function arguments (positional and keyword arguments) + +```yaml +#@ def my_deployment(name, replicas=1, labels={}): +kind: Deployment +metadata: + name: #@ name + labels: #@ labels +spec: + replicas: #@ replicas +#@ end + +--- +kind: List +items: +- #@ my_deployment("dep1", replicas=3) +``` + +- common function usage + +To set `labels` key to return value of `my_labels()`: + +```yaml +labels: #@ my_labels() +labels_as_array: +- #@ my_labels() +``` + +To merge return value of `my_labels()` into `labels` map: + +```yaml +#@ load("@ytt:template", "template") + +labels: + another-label: true + _: #@ template.replace(my_labels()) +``` + +Note that in most cases `template.replace` is not necessary since it's only helps replacing one item (array item, map item or document) with multiple items of that type. diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-dict.md b/site/content/ytt/docs/v0.50.x/lang-ref-dict.md new file mode 100644 index 000000000..e04eaa843 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-dict.md @@ -0,0 +1,80 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-dict] +title: Dictionaries +--- + +```yaml +#@ color = {"red": 123, "yellow": 100, "blue": "245"} +red: #@ color["red"] +``` + +Copied here for convenience from [Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#dictclear). + +- [dict·clear](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·clear) (`D.clear()`) +```python +x = {"one": 1, "two": 2} +x.clear() # None +print(x) # {} +``` + +- [dict·get](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·get) (`D.get(key[, default])`) +```python +x = {"one": 1, "two": 2} +x.get("one") # 1 +x.get("three") # None +x.get("three", 0) # 0 +``` + +- [dict·items](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·items) (`D.items()`) +```python +x = {"one": 1, "two": 2} +x.items() # [("one", 1), ("two", 2)] +``` + +- [dict·keys](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·keys) (`D.keys()`) +```python +x = {"one": 1, "two": 2} +x.keys() # ["one", "two"] +``` + +- [dict·pop](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·pop) (`D.pop(key[, default])`) +```python +x = {"one": 1, "two": 2} +x.pop("one") # 1 +x # {"two": 2} +x.pop("three", 0) # 0 +x.pop("four") # error: missing key +``` + +- [dict·popitem](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·popitem) (`D.popitem()`) +```python +x = {"one": 1, "two": 2} +x.popitem() # ("one", 1) +x.popitem() # ("two", 2) +x.popitem() # error: empty dict +``` + +- [dict·setdefault](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·setdefault) (`D.setdefault(key[, default])`) +```python +x = {"one": 1, "two": 2} +x.setdefault("one") # 1 +x.setdefault("three", 0) # 0 +x # {"one": 1, "two": 2, "three": 0} +x.setdefault("four") # None +x # {"one": 1, "two": 2, "three": None} +``` + +- [dict·update](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·update) (`D.update([pairs][, name=value[, ...])`) +```python +x = {} +x.update([("a", 1), ("b", 2)], c=3) +x.update({"d": 4}) +x.update(e=5) +x # {"a": 1, "b": "2", "c": 3, "d": 4, "e": 5} +``` + +- [dict·values](https://github.com/google/starlark-go/blob/master/doc/spec.md#dict·values) (`D.values()`) +```python +x = {"one": 1, "two": 2} +x.values() # [1, 2] +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-for.md b/site/content/ytt/docs/v0.50.x/lang-ref-for.md new file mode 100644 index 000000000..d73222ef5 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-for.md @@ -0,0 +1,40 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-for] +title: For loop +--- + +Refer to [Starlark for loop specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#for-loops) for details. + +- iterating with values + +```yaml +array: +#@ for i in range(0,3): +- #@ i +- #@ i+1 +#@ end +``` + +- iterating with index + +```yaml +array: +#@ arr = [1,5,{"key":"val"}] +#@ for i in range(len(arr)): +- val: #@ arr[i] + index: #@ i +#@ end +``` + +- use of `continue/break` + +```yaml +array: +#@ for i in range(0,3): +#@ if i == 1: +#@ continue +#@ end +- #@ i +- #@ i+1 +#@ end +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-if.md b/site/content/ytt/docs/v0.50.x/lang-ref-if.md new file mode 100644 index 000000000..914115d3a --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-if.md @@ -0,0 +1,82 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-if] +title: If Statements +--- + +Refer to [Starlark if statement specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#if-statements) for details. + +- if + +```yaml +#@ if True: +test1: 123 +test2: 124 +#@ end +``` + +- if (negative) + +```yaml +#@ if not True: +test1: 123 +#@ end +``` + +- single-node if + +```yaml +#@ if/end True: +test1: 123 +``` + +- if-else conditional + +```yaml +#@ if True: +test1: 123 +#@ else: +test2: 124 +#@ end +``` + +- if-elif-else conditional + +```yaml +#@ if True: +test2: 123 +#@ elif False: +test2: 124 +#@ else: +test2: 125 +#@ end +``` + +- if-elif-else conditional boolean (and/or) \ + See [Starlark or/and operators](https://github.com/google/starlark-go/blob/master/doc/spec.md#or-and-and) for more details. + + +```yaml +#@ test = 123 +#@ if test > 100 and test < 200: +test1: 123 +#@ elif test == 100 or test == 200: +test2: 124 +#@ else: +test3: 125 +#@ end +``` + + +- single line if + +```yaml +#@ passwd = "..." +test1: #@ passwd if passwd else assert.fail("password must be set") +``` + +- implicit if + +```yaml +#@ passwd = "..." +test1: #@ passwd or assert.fail("password must be set") +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-list.md b/site/content/ytt/docs/v0.50.x/lang-ref-list.md new file mode 100644 index 000000000..33661f57b --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-list.md @@ -0,0 +1,60 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-list] +title: Lists +--- + +```yaml +#@ nums = [123, 374, 490] +first: #@ nums[0] +``` + +Copied here for convenience from [Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#listappend). + +- [list·append](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·append) (`L.append(x)`) +```python +x = [] +x.append(1) # None +x.append(2) # None +x.append(3) # None +x # [1, 2, 3] +``` + +- [list·clear](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·clear) (`L.clear()`) +```python +x = [1, 2, 3] +x.clear() # None +x # [] +``` + +- [list·extend](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·extend) (`L.extend(x)`) +```python +x = [] +x.extend([1, 2, 3]) # None +x.extend(["foo"]) # None +x # [1, 2, 3, "foo"] +``` + +- [list·index](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·index) (`L.index(x[, start[, end]])`) +```python +x = list("banana".codepoints()) +x.index("a") # 1 (bAnana) +x.index("a", 2) # 3 (banAna) +x.index("a", -2) # 5 (bananA) +``` + +- [list·insert](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·insert) (`L.insert(i, x)`) +```python +x = ["b", "c", "e"] +x.insert(0, "a") # None +x.insert(-1, "d") # None +x # ["a", "b", "c", "d", "e"] +``` + +- [list·pop](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·pop) (`L.pop([index])`) +- [list·remove](https://github.com/google/starlark-go/blob/master/doc/spec.md#list·remove) (`L.remove(x)`) +```python +x = [1, 2, 3, 2] +x.remove(2) # None (x == [1, 3, 2]) +x.remove(2) # None (x == [1, 3]) +x.remove(2) # error: element not found +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-load.md b/site/content/ytt/docs/v0.50.x/lang-ref-load.md new file mode 100644 index 000000000..a2c764088 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-load.md @@ -0,0 +1,115 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-load] +title: Load Statements +--- + +## Terminology + +- `module`: single file; can export variables, functions, or be templated => some type of result e.g. yaml structure, or string, or None) +- `package`: single directory; contains modules +- `library`: collection of packages + +## Usage + +Load statement allows to load functions from other modules (such as ones from [builtin `ytt` library](lang-ref-ytt.md)). + +- [load](https://github.com/google/starlark-go/blob/master/doc/spec.md#load-statements) +```python +load("@ytt:overlay", "overlay") # load overlay module from builtin ytt library +load("@ytt:overlay", ov="overlay") # load overlay symbol under a different alias +load("helpers.star", "func1", "func2") # load func1, func2 from Starlark file +load("helpers.lib.yml", "func1", "func2") # load func1, func2 from YAML file +load("helpers.lib.txt", "func1", "func2") # load func1, func2 from text file +load("/dir/helpers.lib.yml", "func1") # load func1 from file relative to root of library +load("sub-dir/helpers.lib.txt", "func1") # load func1 from a sub-directory +load("@project:dir/helpers.lib.txt", "func1") # load func1 from a project located under _ytt_lib +``` + +`load` arguments are as follows: + +1. location which takes following shape `[@[library]:][package/]{0,n}module`, where, + - `library` could be `ytt` or local path under `_ytt_lib` directory + - examples: [`ytt`](lang-ref-ytt.md), `github.com/k14s/k8s-lib`, `common` + - `package` could be a directory path + - examples: `overlay`, `regexp`, `app/`, `/app/something` + - `module` is a file name or predefined name (included in `ytt` library) + - examples: `module.lib.yml` +1. one or more symbols to import with optional aliases + - examples: `func1`, `func1="as_func1"` + +Files can be loaded from current or child directories. As of ytt v0.24.0, `/` package prefix can be used to load files relative to the root of the current library. + +Note that there is a distinction between using `load("@project:dir/helpers.lib.txt", "func1")` or `load("@project/dir:helpers.lib.txt", "func1")`, in that, `:` signifies what ytt considers a self-contained library (i.e. is `dir` simply a package in `project` or `project/dir` a standalone library). This allows files to load other files relative to the library root. + +To load a set of functions from a single file, you can create a `struct` that contains references to the functions. For example: +`funcs.star`: +``` +load("@ytt:struct", "struct") + +def testfunc(): + return 123 +end + +def otherfunc(): + return 456 +end + +mod = struct.make(testfunc=testfunc, otherfunc=otherfunc) +``` + +`config.yml`: +``` +#@ load("funcs.star", "mod") + +result: #@ mod.testfunc() +other_result: #@ mod.otherfunc() +``` + +## _ytt_lib directory + +`_ytt_lib` directory allows to keep private dependencies from consumers of libraries. + +For example given following directory structure: + +``` +app1.yml +_ytt_lib/big-corp/sre.lib.yml +_ytt_lib/big-corp/_ytt_lib/big-corp/common/deployments.lib.yml +_ytt_lib/big-corp/_ytt_lib/big-corp/common/services.lib.yml +``` + +- `app1.yml` _can_ load `big-corp/sre.lib.yml` via `@big-corp:sre.lib.yml` +- `app1.yml` _cannot_ load `big-corp/_ytt_lib/big-corp/common/services.lib.yml` as it is a private dependency of anything inside `_ytt_lib/big-corp/` directory (e.g. `sre.lib.yml`) + +hence making it possible for `big-corp/sre.lib.yml` module to keep its `big-corp/common` library dependency private. +## Files + +To make files available to `load` statement they have to be given to ytt CLI via `--file` (`-f`) option. The argument of that option can be a path to either of: + +- a **file**: in which case the file can be loaded by its name. +- a **directory**: in which case all the files found can be loaded by using paths relative to the directory. If the directory contains a `_ytt_lib` folder, then libraries in it can also be loaded. + +For example, given following directory structure: + +``` +app1.yml +helpers.lib.yml +_ytt_lib/apps/apps.lib.yml +sub-dir/more-helpers.lib.yml +sub-dir/_ytt_lib/weird-lib/funcs.lib.yml +``` + +- `ytt -f .` will make it possible for `app1.yml` to load: + - `helpers.lib.yml` + - `@apps:apps.lib.yml` + - `sub-dir/more-helpers.lib.yml` +- `ytt -f helpers.lib.yml -f sub-dir -f app1.yml` will make it possible for `app1.yml` to load: + - `helpers.lib.yml` + - `more-helpers.lib.yml` (not `sub-dir/more-helpers.lib.yml`) + - `@weird-lib:funcs.lib.yml` + +## Examples + +- [Load](/ytt/#example:example-load) +- [Load ytt library](/ytt/#example:example-load-ytt-library-module) +- [Load custom library](/ytt/#example:example-load-custom-library-module) diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-string.md b/site/content/ytt/docs/v0.50.x/lang-ref-string.md new file mode 100644 index 000000000..4be6bbf5b --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-string.md @@ -0,0 +1,130 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-string] +title: Strings +--- + +```yaml +name1: #@ name + "-deployment" +name2: #@ "{}-deployment".format("name") +``` + +Copied here for convenience from [Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#stringelem_ords). + +- [string·elem_ords](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·elem_ords) +- [string·capitalize](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·capitalize) (`S.capitalize()`) +```python +"hello, world!".capitalize() # "Hello, world!"` +``` + +- [string·codepoint_ords](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·codepoint_ords) +- [string·count](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·count) (`S.count(sub[, start[, end]])`) +```python +"hello, world!".count("o") # 2 +"hello, world!".count("o", 7, 12) # 1 (in "world") +``` + +- [string·endswith](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·endswith) (`S.endswith(suffix[, start[, end]])`) +```python +"filename.star".endswith(".star") # True +'foo.cc'.endswith(('.cc', '.h')) # True +``` + +- [string·find](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·find) (`S.find(sub[, start[, end]])`) +```python +"bonbon".find("on") # 1 +"bonbon".find("on", 2) # 4 +"bonbon".find("on", 2, 5) # -1 +``` + +- [string·format](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·format) (`S.format(*args, **kwargs)`) +```python +"a{x}b{y}c{}".format(1, x=2, y=3) # "a2b3c1" +"a{}b{}c".format(1, 2) # "a1b2c" +"({1}, {0})".format("zero", "one") # "(one, zero)" +``` + +- [string·index](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·index) (`S.index(sub[, start[, end]])`) +```python +"bonbon".index("on") # 1 +"bonbon".index("on", 2) # 4 +"bonbon".index("on", 2, 5) # error: substring not found (in "nbo") +``` + +- [string·isalnum](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isalnum) +```python +"base64".isalnum() # True +"Catch-22".isalnum() # False +``` + +- [string·isalpha](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isalpha) +- [string·isdigit](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isdigit) +- [string·islower](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·islower) +- [string·isspace](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isspace) +- [string·istitle](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·istitle) +- [string·isupper](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·isupper) +- [string·join](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·join) (`S.join(iterable)`) +```python +", ".join(["one", "two", "three"]) # "one, two, three" +"a".join("ctmrn".codepoints()) # "catamaran" +``` + +- [string·lower](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·lower) +- [string·lstrip](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·lstrip) +```python +" hello ".lstrip() # "hello " +``` + +- [string·partition](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·partition) +- [string·replace](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·replace) (`S.replace(old, new[, count])`) +```python +"banana".replace("a", "o") # "bonono" +"banana".replace("a", "o", 2) # "bonona" +``` + +- [string·rfind](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rfind) (`S.rfind(sub[, start[, end]])`) +```python +"bonbon".rfind("on") # 4 +"bonbon".rfind("on", None, 5) # 1 +"bonbon".rfind("on", 2, 5) # -1 +``` + +- [string·rindex](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rindex) (`S.rindex(sub[, start[, end]])`) +- [string·rpartition](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rpartition) +- [string·rsplit](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rsplit) +- [string·rstrip](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·rstrip) (`S.rstrip()`) +```python +" hello ".rstrip() # " hello" +``` + +- [string·split](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·split) (`S.split([sep [, maxsplit]])`) +```python +"one two three".split() # ["one", "two", "three"] +"one two three".split(" ") # ["one", "two", "", "three"] +"one two three".split(None, 1) # ["one", "two three"] +"banana".split("n") # ["ba", "a", "a"] +"banana".split("n", 1) # ["ba", "ana"] +``` + +- [string·elems](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·elems) +- [string·codepoints](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·codepoints) +- [string·splitlines](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·splitlines) (`S.splitlines([keepends])`) +```python +"one\n\ntwo".splitlines() # ["one", "", "two"] +"one\n\ntwo".splitlines(True) # ["one\n", "\n", "two"] +``` + +- [string·startswith](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·startswith) (`S.startswith(prefix[, start[, end]])`) +```python +"filename.star".startswith("filename") # True` +``` + +- [string·strip](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·strip) (`S.strip()`) +```python +" hello ".strip() # "hello" +``` + +- [string·title](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·title) +- [string·upper](https://github.com/google/starlark-go/blob/master/doc/spec.md#string·upper) (`S.upper()`) +```python +"Hello, World!".upper() # "HELLO, WORLD!" +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-structs.md b/site/content/ytt/docs/v0.50.x/lang-ref-structs.md new file mode 100644 index 000000000..aa4ca708c --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-structs.md @@ -0,0 +1,94 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-structs] +title: Structs +--- + +## Overview + +Structs are well-defined data objects, comprised of key/value pairs known as "attributes". They are a way to store and refer to data of a known structure. + +The most commonly used `struct` is `data.values`, supplied by the [`@ytt:data`](ytt-data-values.md) module. For example, a data values defined by: + +```yaml +#@data/values +--- +db_conn: + host: acme.example.com +``` + +is automatically processed into a `struct` (named `values`): the keys in the `@data/values` file are defined one-for-one as attributes on the `struct`. + +Those attribues can be referenced by name: + +```yaml +#@ load("@ytt:data", "data") +--- +persistence: + db_url: #@ data.values.db_conn.host +``` + +--- +## Attributes + +Attributes are a key/value pair, where the key is a `string` and the value can be of any type. + +Attributes can be referenced: +- by field (using "[dot notation](https://github.com/google/starlark-go/blob/master/doc/spec.md#dot-expressions)") + ```yaml + db_url: #@ data.values.db_conn.host + ``` +- by string (using "[index notation](https://github.com/google/starlark-go/blob/master/doc/spec.md#index-expressions)") _(as of v0.31.0)_: + ```yaml + db_url: #@ data.values["db_conn"]["host"] + ``` + useful when the name of the attribute is not known statically. + +Referencing an attribute that is not defined on the `struct` results in an error: +```yaml +db_url: #@ data.values.bd_conn # <== struct has no .bd_conn field or method +db_host: #@ data.values["db_conn"]["hast"] # <== key "hast" not in struct +``` + +--- +## Built-in Functions + +The following built-ins can be useful with `struct` values: + +- [`dir()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#dir) enumerates all its attributes. + ``` + load("@ytt:struct", "struct") + + foo = struct.encode({"a": 1, "b": 2, "c": 3}) + keys = dir(foo) # <== ["a", "b", "c"] + ``` + +- [`getattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#getattr) can be used to select an attribute. + ```yaml + #@ for vpc_config in getattr(getattr(data.values.accounts, foundation_name), vpc_name): + ... + #@ end + ``` + - as of v0.31.0, `struct`s support [index notation](https://github.com/google/starlark-go/blob/master/doc/spec.md#index-expressions) behaving identically, more succinctly/readably: + ```yaml + #@ for vpc_config in data.values.accounts[foundation_name][vpc_name]: + ... + #@ end + ``` + +- [`hasattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#hasattr) reports whether a value has a specific attribute + ```python + # `value` is a struct that _might_ have a field, "additional_ports" + + def ports(value): + if hasattr(value, "additional_ports"): + ... + end + ``` + +--- +## Creating structs + +`struct` instances can be made using the `@ytt:struct` module. + +See [ytt Library: struct module](lang-ref-ytt-struct.md) for details. + diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-yaml-fragment.md b/site/content/ytt/docs/v0.50.x/lang-ref-yaml-fragment.md new file mode 100644 index 000000000..547c8a7b8 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-yaml-fragment.md @@ -0,0 +1,104 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-yaml-fragment] +title: YAMLFragments +--- + +## Overview + +YAMLFragment is a type of value that is defined directly in YAML (instead of plain Starlark). For example, function `val()` returns a value of type `yamlfragment`. + +```yaml +#@ def vals(): +key1: val1 +key2: + subkey: val2 +#@ end +``` + +YAMLFragment may contain: + +- YAML document set (array of YAML documents) +- YAML array +- YAML map +- null + +Given various contents it wraps, YAMLFragment currently exposes limited ways of accessing its contents directly. Following accessors are available in v0.26.0+. + +## YAML Document Set + +```yaml +#@ def docs(): +--- +doc1 +--- +doc2 +--- +doc3 +#@ end +``` + +- access contents of a document at particular index +```python +docs()[1] # returns "doc2" +``` + +- loop over each document, setting `val` to its contents +```python +for val in docs(): + val # ... +end +``` + +## YAML Array + +```yaml +#@ def vals(): +- val1 +- val2 +#@ end +``` + +- access contents of an array item at particular index +```python +vals()[1] # returns "val2" +``` + +- loop over each array item, setting `val` to its contents +```python +for val in vals(): + val # ... +end +``` + +## YAML Map + +```yaml +#@ def vals(): +key1: val1 +key2: + subkey: val2 +#@ end +``` + +- access contents of a map item with particular key +```python +vals()["key1"] # returns "val1" +``` + +- check if map contains particular key +```python +"key1" in vals() # returns True +"key6" in vals() # returns False +``` + +- loop over each map item, setting `val` to its contents +```python +for key in vals(): + val = vals()[key] # ... +end +``` + +- convert to a dictionary +```python +dict(**vals()) # returns {"key1": "val1", "key2": yamlfragment({"subkey": "val2"})} +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-assert.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-assert.md new file mode 100644 index 000000000..a5f3c3ae3 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-assert.md @@ -0,0 +1,160 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-assert] +title: Assert Module +--- + +## Overview + +`ytt`'s Assert module allows users to make assertions about their templates and stop execution if desired. + +## Functions + +The `@ytt:assert` module provides several built-in assertion functions. +To use these functions, include the `@ytt:assert` module: + +```python +#@ load("@ytt:assert", "assert") +``` + +### assert.equals() + - Checks equality of the two arguments provided, stops execution if values are not equal. +```python +load("@ytt:assert", "assert") + +assert.equals("not", "equal") # stops execution +``` + +### assert.fail() +- Stops execution and reports a failure. +- Takes a single string argument used as the failure message, this can be formatted with available values. +```python +load("@ytt:assert", "assert") + +assert.fail("custom failure message") +assert.fail("expected value foo, but was {}".format(value)) +x = data.values.env.mysql_password or assert.fail("missing env.mysql_password") +``` + +### assert.max() + +- Checks that values are less than or equal to the maximum value. +- `x = assert.max(4)` returns an object, `x`, that can `x.check()` if values are less than or equal to 4. +- Is able to compare numbers, strings, lists, dictionaries, and [YAML fragments](lang-ref-yaml-fragment). +```python +load("@ytt:assert", "assert") + +assert.max(4).check(3) + +assert.max(4).check(5) # stops execution +``` + +### assert.max_len() + +- Checks that values have length less than or equal to the maximum length. +- Maximum length argument is an integer. +- `x = assert.max_len(4)` creates an object, `x` that can `check()` if the length of values are less than or equal to 4. +- Checks the length of strings, lists, dictionaries, and [YAML fragments](lang-ref-yaml-fragment). +```python +load("@ytt:assert", "assert") + +assert.max_len(4).check({'foo': 0, 'bar': 1}) + +assert.max(4)_len.check("123.45.67.89") # stops execution +``` + +### assert.min() + +- Checks that values are greater than or equal to the minimum value. +- `x = assert.min(2)` returns an object, `x`, that can `x.check()` if values are greater than or equal to 2. +- Is able to compare numbers, strings, lists, dictionaries, and [YAML fragments](lang-ref-yaml-fragment). +```python +load("@ytt:assert", "assert") + +assert.min(2).check(5) + +assert.min(2).check(1) # stops execution +``` + +### assert.min_len() + + - Checks that values have length greater than or equal to the minimum length. + - Minimum length argument is an integer. + - `x = assert.min_len(1)` creates an object, `x` that can `check()` if the length of values are greater than or equal to 1. + - Checks the length of strings, lists, dictionaries, and [YAML fragments](lang-ref-yaml-fragment). +```python +load("@ytt:assert", "assert") + +assert.min_len(1).check(["some","list","of","values"]) + +assert.min_len(1).check("") # stops execution +``` + +### assert.not_null() + + - Checks that a value is not null or none. +```python +load("@ytt:assert", "assert") + +v = None +assert.not_null(v) # stops execution +# is syntactic sugar for +assert.not_null().check(v) # stops execution +``` + +### assert.one_not_null() + +- Checks that a map (or dictionary)'s value has one and only one not-null item. +```python +load("@ytt:assert", "assert") + +# passes +assert.one_not_null().check({"foo": 1, "bar": None}) + +# fails: two values are not null (stops execution) +assert.one_not_null().check({"foo": 1, "bar": 2}) + +# passes: one of named values is not null +assert.one_not_null(["foo", "bar"]).check({"foo": 1, "bar": None, "baz": 3}) + +# passes: missing keys are ok +assert.one_not_null(["foo", "not-present"]).check({"foo": 1, "bar": 2}) +``` + +### assert.one_of() + +- Checks that the value is one in the specified list +```python +load("@ytt:assert", "assert") + +# passes +assert.one_of(["debug", "info", "warn"]).check("warn") + +# fails: value not in the list. +assert.one_of(["aws", "azure", "gcp"]).check("digitalocean") + +# An assertion can be used multiple times against different values +valid_ports = assert.one_of([1433, 1434, 1521, 1830, 3306, 5432]) + +# all pass +valid_ports.check(3306) +valid_ports.check(5432) + +# fails: items in enumeration are integers, value is a string +valid_ports.check("5432") +``` + +### assert.try_to() +- Invokes a function, catching failure if one occurs. +- Takes single function as argument. +- Returns the return value of function, or the error if one occurs. +```python +load("@ytt:assert", "assert") + +x, err = assert.try_to(lambda : json.decode('{"key": "value"}')) +x # { "key" = "value" } (i.e. dict with one entry) +err # None + +x, err = assert.try_to(lambda : json.decode("(not JSON)")) +x # None +err # "json.decode: invalid character '(' looking for beginning of value" +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-library.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-library.md new file mode 100644 index 000000000..3a7a312fb --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-library.md @@ -0,0 +1,400 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-library] +title: Library Module +--- + +## Overview + +You can extract a whole set of input files (i.e. templates, overlays, data values, etc.) into a "Library". + +For example: + +```yaml +config/ +├── _ytt_lib/ +│ └── frontend/ +│ ├── schema.yml +│ └── store.yml +└── config.yml +``` +where: +- `config/_ytt_lib/frontend/` and its contents is a library named `"frontend"` + +Libraries are _not_ automatically included in `ytt` output; one must programmatically load, configure, evaluate, and insert those results into a template that is part of the output. + +```yaml +#! config/config.yml -- example of using a library + +#@ load("@ytt:library", "library") +#@ load("@ytt:template", "template") + +#! 1. Load an instance of the library +#@ app = library.get("frontend") + +#! 2. Create a configured copy of the library (does not mutate original) +#@ app_with_vals = app.with_data_values({"apiDomain": "gateway.example.com"}) + +#! 3. Evaluate the library and include results (a document set) in the output +--- #@ template.replace(app_with_vals.eval()) +``` + +For a complete working example, see [ytt-library-module example](/ytt/#example:example-ytt-library-module). + +## What is a Library? + +A `ytt` library is a directory tree contained within a specially-named directory: [`_ytt_lib/`](lang-ref-load.md#_ytt_lib-directory). +- The library's _name_ is the path relative from the `_ytt_lib/` directory. +- The library's _contents_ are those of the directory along with subdirectories, recursively. +- A library may contain libraries as well, if one of its subdirectories is `_ytt_lib/`. + +The root directory of a `ytt` invocation is itself a library known as the "root library". + +Libraries are evaluated in isolation: each a separate execution of the pipeline described in [How it works](how-it-works.md). +- Each library has its own data values schema. +- Overlays within a library only apply over _its_ evaluated document set. +- The final evaluated result is returned as a [YAML Fragment wrapping a document set.](lang-ref-yaml-fragment.md#yaml-document-set). + +--- + +## Functions + +There's but one function in the `@ytt:library` module: [`library.get()`](#libraryget) + +### library.get() + +Contructs a new [`@ytt:library.instance`](#library-instances) based on the contents from the named library. + +```python +instance = library.get(name, []) +``` + +- **`name`** (`string`) — path to the base directory of the desired library: `./_ytt_lib/`. Can contain slashes `/` for sub-directories (e.g. `github.com/carvel-dev/ytt-library-for-kubernetes/app`) +- keyword arguments (optional): + - **`alias=`** (`string`) — unique name for this library instance. See [Aliases](#aliases), below. + - **`ignore_unknown_comments=`** (`bool`) — equivalent to `ytt --ignore-unknown-comments`; see [File Marks > type detection for YAML files](file-marks.md#type-detection-for-yaml-files) for more details (default: `False`). (as of v0.31.0) + - **`implicit_map_key_overrides=`** (`bool`) — equivalent to `ytt --implicit-map-key-overrides`; see [@yaml/map-key-override](lang-ref-annotation.md#yaml-templating-annotations) for more details. (default: `False`). (as of v0.31.0) + - **`strict=`** (`bool`) — equivalent to `ytt --strict` (default: `False`). (as of v0.31.0) +- **`instance`** ([`@ytt:library.instance`](#library-instances)) — a new library instance backed by the contents of the named library. + +The file containing this method invocation must be a sibling of the [`_ytt_lib` directory](lang-ref-load.md#_ytt_lib-directory). + +--- + +## Library Instances + +Each library returned from a function within this module is a copy: a separate instance. + +A library instance (a value of type `@ytt:library.instance`) is created from source with [`library.get()`](#libraryget). + +With a library instance: +- create configured copies using: + - [`instance.with_data_values_schema()`](#instancewith_data_values_schema) + - [`instance.with_data_values()`](#instancewith_data_values) +- evaluate its contents via [`instance.eval()`](#instanceeval) +- fetch values from it using: + - [`instance.data_values()`](#instancedata_values) for the final data values for the library + - [`instance.export()`](#instanceexport) to access its functions and variables + +### instance.data_values() + +Calculates and returns just the Data Values configured on this library instance. + +```python +dvs = instance.data_values() +``` + +- **`dvs`** ([`struct`](lang-ref-structs.md)) — the final data values (i.e. the net result of [all configured data values](ytt-data-values.md#library-data-values)). + +### instance.eval() + +Calculates the library's final data values (i.e. the net result of [all configured data values](ytt-data-values.md#library-data-values)), evaluates its templates into a document set, and applies its overlays on that document set (i.e. executes the pipeline described in [How it works](how-it-works.md) for this library instance's inputs and contents. The output of that execution — rather than rendered automatically — is returned in a variable). + +```python +document_set = instance.eval() +``` + +- **`document_set`** ([`yamlfragment`](lang-ref-yaml-fragment.md#yaml-document-set)) — the YAML document set resulting from the evaluation of this instance. + +Note: the resulting Document Set is _**not**_ automatically included in output. A common way to include this result in the output is to use [`template.replace()`](lang-ref-ytt-template.md#templatereplace): + +```yaml +#@ load("@ytt:template", "template") +#@ load("@ytt:library", "library") + +--- #@ template.replace(library.get("cert-manager").eval()) +``` + +See also, [Playground: ytt library module](/ytt/#example:example-ytt-library-module) example. + +### instance.export() + +(As of v0.28.0) + +Returns the value of an identifier declared within the library instance. + +```python +value = instance.export(name, [path=]) +``` + +- **`name`** (`string`) — the name of a function or a variable declared within some [module](lang-ref-load.md#terminology)/file in the library. (i.e. a file with the extension `.lib.yml` or `.star`). +- **`path=`** (`string`) — the path to the module/file that contains the declaration. Only required when `name` is not unique within the library. +- **`value`** (any) — a copy of the specified value. + - if `value` is a function, it is executed within the context of _its_ library instance. For example, if the function depends on values from the [`@ytt:data`](lang-ref-ytt.md#data) module, the values provided are those of this library instance. + +**Examples:** + +_Example 1: Exporting a function from a library._ + +Assuming some module/file in the "helpers" library contains the definition: + +```python +... +def wrap_name(name): + ... +end +... +``` + +Can be exported and used from another library: + +```python +helpers = library.get("helpers") +wrap_name = helpers.export("wrap_name") + +full_name = wrap_name("app") +``` + +_Example 2: Disambiguating between multiple declarations of function._ + +Assuming two modules/files in the "helpers" library have the same name: + +```python +# main/funcs.star +def wrap_name(name): ... +``` +and +```python +# lib/funcs.star +def wrap_name(name): ... +``` + +One of which can be unambiguously referenced: + +```python +helpers = library.get("helpers") +wrap_name = helpers.export("wrap_name", path="lib/funcs.star") + +full_name = wrap_name("app") +``` + +Note: without the `path=` keyword argument, `helpers.export()` would report an error. + +### instance.with_data_values() + +Returns a copy of the library instance with data values overlayed with those given. + +```python +new_instance = instance.with_data_values(dvs, [plain=]) +``` + +- **`dvs`** (`struct` | [`yamlfragment`](lang-ref-yaml-fragment.md)) — data values with which to overlay (or set, if none exist). + - only `yamlfragment`s wrapping a map or an array are supported (i.e. `yamlfragment`s wrapping document sets are not supported). + - `yamlfragment` values _can_ contain [overlay annotations](lang-ref-ytt-overlay.md#overlay-annotations) for fine-grained overlay control. +- **`plain=`** (`bool`) — when `True` indicates that `dvs` should be "plain merged" over existing data values (i.e. the exact same behavior as [`--data-values-file`](ytt-data-values.md#configuring-data-values-via-command-line-flags)). + - `dvs` must be plain YAML (i.e. a `struct` or a `yamlfragment` with no annotations). +- **`new_instance`** ([`@ytt:library.instance`](#library-instances)) — a copy of `instance` with `dvs` overlayed on its data values; `instance` remains unchanged. + +### instance.with_data_values_schema() + +(As of v0.35.0) + +Returns a copy of the library instance with data values schema overlayed with that given. + +```python +new_instance = instance.with_data_values_schema(schema) +``` + +- **`schema`** (`struct` | [`yamlfragment`](lang-ref-yaml-fragment.md)) — schema for data values with which to overlay on existing schema (or set if none exist). + - only `yamlfragment`s wrapping a map or an array are supported (i.e. `yamlfragment`s wrapping document sets are not supported) + - `yamlfragment` values _can_ contain [overlay annotations](lang-ref-ytt-overlay.md#overlay-annotations) for fine-grained overlay control. +- **`new_instance`** ([`@ytt:library.instance`](#library-instances)) — a copy of `instance` with a schema updated with `schema`; `instance` remains unchanged. + +**Examples:** + +_Example 1: Declaring a new data value (and setting it)._ + +```yaml +#@ def app_schema(): +name: "" +#@overlay/match missing_ok=True +env_vars: + custom_key: "" +#@ end + +#@ app1_with_schema = app1.with_data_values_schema(app_schema()) +--- +#@ def app_vals(): +name: app1 +env_vars: + custom_key: some_val +#@ end + +#@ app1_with_vals = app1.with_data_values(app_vals()) +``` + +--- + +## Annotations + +### @library/ref + +(As of v0.28.0) + +Attaches a YAML document to the specified library. When the library is evaluated, the annotated document is included. +Only supported on documents annotated with `@data/values` and `@data/values-schema`. + +``` +@library/ref library_name +``` +- **`library_name`** (`string`) — `@`-prefixed path to the base directory of the desired library: `./_ytt_lib/`. Can contain slashes `/` for sub-directories (e.g. `github.com/carvel-dev/ytt-library-for-kubernetes/app`). Can also be an [alias](#aliases) for specific library instance(s). + +**Examples:** + +_Example 1: Change schema default for a data value in a library._ + +```yaml +#@data/values-schema +#@library/ref "@frontend" +--- +name: "custom" +``` + +Overlays the default value for `name` in the "frontend" library to be "custom". + +_Example 2: Target a data value overlay to a library._ + +```yaml +#@data/values +#@library/ref "@backend" +--- +#@overlay/replace +domains: +- internal.example.com +- internal-backup.example.com +``` + +Sets the "backend" library's `domains` data value to be exactly the values given. + +See also: [Data Values > Setting Library Values via Files](ytt-data-values.md#setting-library-values-via-files). + +Note: data values may also be attached to libraries via [command line flags](ytt-data-values.md#setting-library-values-via-command-line-flags). + +--- + +## Aliases + +To facilitate configuring specific library instances, one can mark them with an alias. + +An alias: +- is defined in a [`library.get()`](#libraryget) call, using the optional `alias=` keyword argument. +- is added to a library reference by prefixing it with a tilde, `~`: + - `@~` refers to _any_ library instance with the alias. + - `@~` refers to any instance of the named library that _also_ has the alias. + +For example, given a library known as "fruit": + +``` +├── apple-values.yml +├── config.yml +├── orange-values.yml +└── _ytt_lib + └── fruit + ├── doc.yml + └── values.yml +``` + +where: +```yaml +#! _ytt_lib/fruit/doc.yml + +#@ load("@ytt:data", "data") +--- #@ data.values +``` +the template in the library simply returns its data values as a document, and ... +```yaml +#! _ytt_lib/fruit/values.yml + +#@data/values +--- +variety: ordinary +poisoned: false +``` +... those are the data values in the library. + +The root library can assign aliases to library instances: + +```yaml +#! ./config.yml + +#@ load("@ytt:library", "library") + +#@ apple1 = library.get("fruit", alias="apple") +#@ apple2 = apple1.with_data_values({"variety": "jonamac"}) + +#@ orange = library.get("fruit", alias="orange") + +--- +apple: + 1: #@ apple1.eval()[0] + 2: #@ apple2.eval()[0] +orange: #@ orange.eval()[0] +``` +where: +- `apple1` has the alias "apple" +- `apple2` also has the alias "apple" (part of being a copy of `apple1`) +- `orange` has the alias "orange" + +These aliases can be used to target changes to specific library instance(s). + +For example, our root library has these two data values overlays: + +```yaml +#! ./apple-values.yml + +#@data/values +#@library/ref "@~apple" +--- +variety: red delicious +poisoned: true +``` +... which will affect all library instances with the alias "apple", and ... + +```yaml +#! ./orange-values.yml + +#@data/values +#@library/ref "@~orange" +--- +variety: valencia +``` + +... overlays on top of library instance with the alias "orange". + +When the whole fileset is evaluated, the result is: + +```yaml +apple: + 1: + variety: red delicious + poisoned: true + 2: + variety: jonamac + poisoned: true +orange: + variety: valencia + poisoned: false +``` + +notice: +- only the "@~orange" instance has the variety = "valencia" +- both "@~apple" library instances are poisoned; while the "orange" instance is not. diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-overlay.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-overlay.md new file mode 100644 index 000000000..7dff6289d --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-overlay.md @@ -0,0 +1,759 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-overlay] +title: Overlay module +--- + +## Overview + +`ytt`'s Overlay feature provides a way to combine YAML structures together with the help of annotations. + +There are two (2) structures involved in an overlay operation: +- the "left" — the YAML document(s) (and/or contained maps and arrays) being modified, and +- the "right" — the YAML document (and/or contained maps and arrays) that is the overlay, describing the modification. + +Each modification is composed of: +- a matcher (via an [`@overlay/(match)`](#matching-annotations) annotation), identifying which node(s) on the "left" are the target(s) of the edit, and +- an action (via an [`@overlay/(action)`](#action-annotations) annotation), describing the edit. + +Once written, an overlay can be applied in one of two ways: +- on all rendered templates, [**declaratively**, via YAML documents annotated with `@overlay/match`](#overlays-as-files); this is the most common approach. +- on selected documents, [**programmatically**, via `overlay.apply()`](#programmatic-access). + +Programmatic overlays are applied immediately (while the contained template is being evaluated). +Declarative overlays are applied in the "Apply Overlays" step as described in [How it works](how-it-works.md). + +_(For a step-by-step primer on writing and using overlays, watch [Primer on `ytt` Overlays](/blog/primer-on-ytt-overlays/).)_ + +--- +## Overlays as files + +As `ytt` scans input files, it pulls aside any YAML Document that is annotated with `@overlay/match`, and considers it an overlay. + +After YAML templates are rendered, the collection of identified overlays are applied. Each overlay executes, one-at-a-time over the entire set of the rendered YAML documents. + +Order matters: modifications from earlier overlays are seen by later overlays. Overlays are applied in the order detailed in [Overlay order](#overlay-order), below. + +In the example below, the last YAML document is an overlay (it has the `@overlay/match` annotation). +That overlay matcher's selects the first YAML document *only*: it's the only one that has a `metadata.name` of `example-ingress`. + +```yaml +#@ load("@ytt:overlay", "overlay") + +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: example-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: another-example-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / + +#@overlay/match by=overlay.subset({"metadata":{"name":"example-ingress"}}) +--- +metadata: + annotations: + #@overlay/remove + ingress.kubernetes.io/rewrite-target: +``` + +yields: + +```yaml +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: example-ingress + annotations: {} +--- +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: another-example-ingress + annotations: + ingress.kubernetes.io/rewrite-target: / +``` + +See also: [Overlay files example](/ytt/#example:example-overlay-files) in online playground. + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +### Overlay order + +(as of v0.13.0) + +Overlays are applied, in sequence, by: + +1. left-to-right for file flags + - e.g. in `-f overlay1.yml -f overlay2.yml`, `overlay1.yml` will be applied first +1. if file flag is set to a directory, files are alphanumerically sorted + - e.g. in `aaa/z.yml xxx/c.yml d.yml`, will be applied in following order `aaa/z.yml d.yml xxx/c.yml` +1. top-to-bottom order for overlay YAML documents within a single file + +### Next Steps + +Familiarize yourself with the [overlay annotations](#overlay-annotations). + +--- +## Programmatic access + +Overlays need not apply to the entire set of rendered YAML documents (as is the case with [the declarative approach](#overlays-as-files)). + +Instead, the declared modifications can be captured in a function and applied to a specific set of documents via [`overlay.apply()`](#overlayapply) in Starlark code. + +In this example we have `left()` function that returns target structure and `right()` that returns the overlay, specifying the modifications. + +`overlay.apply(...)` will execute the execute the overlay and return a new structure. + +```yaml +#@ load("@ytt:overlay", "overlay") + +#@ def left(): +key1: val1 +key2: + key3: + key4: val4 + key5: + - name: item1 + key6: val6 + - name: item2 + key7: val7 +#@ end + +#@ def right(): +#@overlay/remove +key1: val1 +key2: + key3: + key4: val4 + key5: + #@overlay/match by="name" + - name: item2 + #@overlay/match missing_ok=True + key8: new-val8 +#@ end + +result: #@ overlay.apply(left(), right()) +``` + +yields: + +```yaml +result: + key2: + key3: + key4: val4 + key5: + - name: item1 + key6: val6 + - name: item2 + key7: val7 + key8: new-val8 +``` + +### Next Steps + +Familiarize yourself with the two kinds of overlay annotations: +- matchers (via an [`@overlay/(match)`](#matching-annotations) annotation), and +- actions (via an [`@overlay/(action)`](#action-annotations) annotation). + +--- +## `@overlay` Annotations + +There are two groups of overlay annotations: + +- [Matching Annotations](#matching-annotations) +- [Action Annotations](#action-annotations) + +--- +## Matching Annotations + +These annotations are used to select which structure(s) will be modified: + +- [@overlay/match](#overlaymatch) +- [@overlay/match-child-defaults](#overlaymatch-child-defaults) + +### @overlay/match + +Specifies which nodes on the "left" to modify. + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/match [by=Function|String, expects=Int|String|List|Function, missing_ok=Bool, when=Int|String|List] +``` +- **`by=`**`Function|String` — criteria for matching nodes on the "left" + - `Function` — predicate of whether a given node is a match + - provided matcher functions (supplied by the `@ytt:overlay` module): + - [`overlay.all()`](#overlayall) + - [`overlay.subset()`](#overlaysubset) + - [`overlay.index()`](#overlayindex) + - [`overlay.map_key()`](#overlaymap_key) + - [Custom matcher function](#custom-overlay-matcher-functions) can also be used + - `String` — short-hand for [`overlay.map_key()`](#overlaymap_key) with the same argument + - Defaults (depends on the type of the annotated node): + - document or array item: none (i.e. `by` is required) + - map item: key equality (i.e. [`overlay.map_key()`](#overlaymap_key)) +- **`expects=`**`Int|String|List|Function` — (optional) expected number of nodes to be found in the "left." If not satisfied, raises an error. + - `Int` — must match this number, exactly + - `String` (e.g. `"1+"`) — must match _at least_ the number + - `Function(found):Bool` — predicate of whether the expected number of matches were found + - `found` (`Int`) — number of actual matches + - `List[Int|String|Function]` — must match one of the given criteria + - Default: `1` (`Int`) (i.e. expecting to match exactly one (1) node on the "left"). +- **`missing_ok=`**`Bool` (optional) shorthand syntax for `expects=[0,1]` +- **`when=`**`Int|String|List` (optional) criteria for when the overlay should apply. If the criteria is met, the overlay applies; otherwise, nothing happens. + - `Int` — must equal this number, exactly + - `String` (e.g. `"1+"`) — must match _at least_ the number + - `List[Int|String]` — must match one of the given criteria + +**Notes:** +- `expects`, `missing_ok`, and `when` are mutually-exclusive parameters. +- take care when `expects` includes zero (0); matching none is indistinguishable from a mistakenly written match (e.g. a misspelling of a key name) + +**Examples:** + +- `#@overlay/match by="id"`: expects to find one (1) node on the "left" that has the key `id` and value matching the same-named item on the "right." +- `#@overlay/match by=`[`overlay.map_key("name")`](#overlaymap_key): (same as above) +- `#@overlay/match by=`[`overlay.all`](#overlayall)`,expects="0+"`: has no effective matching expectations +- `#@overlay/match missing_ok=True`: expects to find 0 or 1 matching nodes on the "left" +- `#@overlay/match expects=2`: expects to find exactly two (2) matching nodes on the "left" +- `#@overlay/match expects="2+"`: expects to find two (2) or more matching nodes on the "left" +- `#@overlay/match expects=[0,1,4]`: expects to find 0, 1 or 4 matching nodes on the "left" +- `#@overlay/match expects=lambda x: return x < 10`: expects 9 or fewer matching nodes on the "left" +- `#@overlay/match when=2`: applies changes only if two (2) nodes are found on the "left" and will not error otherwise +- `#@overlay/match when=2+`: applies changes only if two (2) or more nodes are found on the "left" and will not error otherwise +- `#@overlay/match when=[0,1,4]`: applies changes only if there were exactly 0, 1 or 4 nodes found on the "left" and will not error otherwise + +**History:** +- v0.28.0+ — added `when` keyword argument. + +#### Custom Overlay Matcher Functions + +The matcher functions from `@ytt:overlay` cover many use-cases. From time-to-time, more precise matching is required. + +A matcher function has the following signature: + +`Function(indexOrKey,left,right):Boolean` + - `indexOrKey` + - when `left` is an array item: (`Int`) — the potential match's _zero-based index_ in the list that is the left-side + - when `left` is a map item: (`any`) — the potential match's _key_ in the map that is the left-side + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the potential match/target node + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node in the overlay + - returns `True` if `left` should be considered a match; `False` otherwise. + +Most custom matchers can be written as a Lambda expression. + +Lambda expressions start with the keyword `lambda`, followed by a parameter list, then a `:`, and a single expression that is the body of the function. + +**Examples:** + +_Example 1: Key presence or partial string match_ + +`left` contains `right`: +```python +lambda indexOrKey, left, right: right in left +``` +Returns `True` when `right` is "a member of" `left` + +(see also: [Starlark Spec: Membership tests](https://github.com/google/starlark-go/blob/master/doc/spec.md#membership-tests) for more details) + +_Example 2: Precise string matching_ + +`left` contains a key of the same name as the value of `right`: +```python +lambda indexOrKey, left, right: left["metadata"]["name"].endswith("server") +``` +See also: +- [Language: String](lang-ref-string.md) for more built-in functions on strings. +- [@ytt:regexp Library](lang-ref-ytt.md#regexp) for regular expression matching. + +_Example 3: Match on both key and value_ + +Is the map item with key/value of: `type: LoadBalancer`. + +```python +lambda indexOrKey, left, right: indexOrKey == "type" and left == "LoadBalancer" +``` + +### @overlay/match-child-defaults + +Sets default values for `expects`, `missing_ok`, or `when` for the children of the annotated node. +Does not set these values for the annotated node, itself. + +Commonly used to avoid repeating `@overlay/match missing_ok=True` on each child node in maps. + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/match-child-defaults [expects=Int|String|List|Function, missing_ok=Bool, when=Int|String|List] +``` + +_(see [@overlay/match](#overlaymatch) for parameter specifications.)_ + +**Examples:** + +Without the `#@overlay/match-child-defaults`, _each_ of the four new annotation would have needed an `@overlay/match missing_ok=True` to apply successfully: +```yaml +--- +metadata: + annotations: + ingress.kubernetes.io/rewrite-target: / + +#@overlay/match by=overlay.all +--- +metadata: + #@overlay/match-child-defaults missing_ok=True + annotations: + nginx.ingress.kubernetes.io/limit-rps: 2000 + nginx.ingress.kubernetes.io/enable-access-log: "true" + nginx.ingress.kubernetes.io/canary: "true" + nginx.ingress.kubernetes.io/client-body-buffer-size: 1M +``` + +--- +## Action Annotations + +The following annotations describe how to modify the matched "left" node. + +They are: +- [`@overlay/merge`](#overlaymerge) — (default) combine left and right nodes +- [`@overlay/remove`](#overlayremove) — delete nodes from left +- [`@overlay/replace`](#overlayreplace) — replace the left node +- [`@overlay/insert`](#overlayinsert) — insert right node into left +- [`@overlay/append`](#overlayappend) — add right node at end of collection on left +- [`@overlay/assert`](#overlayassert) — declare an invariant on the left node + +### @overlay/merge + +Merge the value of "right" node with the corresponding "left" node. + +**Valid on:** Map Item, Array Item. + +``` +@overlay/merge +``` +_(this annotation has no parameters.)_ + +**Note:** This is the default action; for each node in an overlay, either the action is explicitly specified or it is `merge`. + +### @overlay/remove + +Deletes the matched "left" node. + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/remove +``` +_(this annotation has no parameters.)_ + +### @overlay/replace + +Substitutes matched "left" node with the value of the "right" node (or by that of a provided function). + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/replace [or_add=Bool, via=Function] +``` + +- **`or_add`**`Bool` _(optional)_ determines what should be done in case matched node is not found + - `or_add=True` indicates that node should be added. Can be used in combination with `@overlay/match missing_ok=True`. +- **`via=`**`Function(left, right): (any)` _(optional)_ determines the value to substitute in. If omitted, the value is `right`. + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the matched node's value + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node + +**History:** +- v0.26.0 — works with [`yamlfragment`](lang-ref-yaml-fragment.md) values. + +**Examples:** + +_Example 1: Use value from "right"_ + +Replaces the corresponding "left" with the value `"v1"` +```yaml +#@overlay/replace +apiVersion: v1 +``` + +_Example 2: Edit string value_ + +```yaml +#@overlay/replace via=lambda left, right: "prefix-"+left +``` + +See also: +- `ytt` modules that export functions useful for manipulating values: + - [base64 module](lang-ref-ytt.md#base64) + - [json module](lang-ref-ytt.md#json) + - [md5 module](lang-ref-ytt.md#md5) + - [sha256 module](lang-ref-ytt.md#sha256) + - [url module](lang-ref-ytt.md#url) + - [yaml module](lang-ref-ytt.md#yaml) +- [Language: String](lang-ref-string.md) for built-in string functions. +- Other Starlark language features that manipulate values: + - [string interpolation](https://github.com/google/starlark-go/blob/master/doc/spec.md#string-interpolation) + - [conditional expressions](https://github.com/google/starlark-go/blob/master/doc/spec.md#conditional-expressions) + - [index expressions](https://github.com/google/starlark-go/blob/master/doc/spec.md#index-expressions) + - [slice expressions](https://github.com/google/starlark-go/blob/master/doc/spec.md#slice-expressions) + +### @overlay/insert + +Inserts "right" node before/after the matched "left" node. The inserted node is either the "right" node or that provided by a function. + +**Valid on:** Document, Array Item. + +``` +@overlay/insert [before=Bool, after=Bool] +``` +- **`before=`**`Bool` whether to insert the "right" node immediately in front of the matched "left" node. +- **`after=`**`Bool` whether to insert the "right" node immediately following the matched "left" node. +- **`via=`**`Function(left, right): (any)` _(optional)_ determines the value to substitute in. If omitted, the value is `right`. + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the matched node's value + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node + +**Examples:** + +Add a `ConfigMap` into each `Namespace`: +```yaml +#@ def configMap(namespace): +apiVersion: v1 +kind: ConfigMap +metadata: + name: insert + namespace: #@ namespace.metadata.name +#@ end + +#@overlay/match overlay.subset({"kind": "Namespace"}) +#@overlay/insert after=True, via=lambda namespace, _: configMap(namespace) +--- +``` + +### @overlay/append + +Inserts the "right" node after the last "left" node. + +**Valid on:** Document, Array Item. + +``` +@overlay/append +``` +_(this annotation has no parameters.)_ + +**Note:** This action implies an `@overlay/match` selecting the last node. Any other `@overlay/match` annotation is ignored. + +**History:** +- v0.32.0 — omitting `@overlay/append` annotation implies a merge operation and 0 matches, defaulting to insert after the last item. + +### @overlay/assert + +Checks assertion that value of "left" matched node equals that of the annotated "right" node (_or_ a provided predicate). + +**Valid on:** Document, Map Item, Array Item. + +``` +@overlay/assert [via=Function] +``` + +- Default: checks that the value of the matched "left" node equals the value of the annotated "right" node. +- **`via`**`=Function(left, right):(Bool|Tuple(Bool|String)|None)` _(optional)_ predicate indicating whether "left" passes the check + - `left` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the matched node's value + - `right` ([`yamlfragment`](lang-ref-yaml-fragment.md) or scalar) — the value of the annotated node + - Return types: + - `Bool` — if `False`, the assertion fails; otherwise, nothing happens. + - `Tuple(Bool|String)` — if `False`, the assertion fails and specified string is appended to the resulting error message; otherwise nothing happens. + - `None` — the assertion assumes to succeed. In these situations, the function makes use of the [`@ytt:assert`](lang-ref-ytt.md#assert) module to effect the assertion. + +**History:** +- v0.26.0 — works with [`yamlfragment`](lang-ref-yaml-fragment.md) values. +- v0.24.0 — introduced + +**Examples:** + +_Example 1: Range check_ + +Fails the execution if `left` not between 0 and 1000, exclusively. + +```yaml +#@overlay/assert via=lambda left, right: left > 0 and left < 1000 +``` + +_Example 2: Well-formedness check_ + +Fails the execution if `left` contains anything other than lowercase letters or numbers. + +```yaml +#@overlay/assert via=lambda left, right: regexp.match("[a-z0-9]+", left) +``` + +See also: +- `ytt` hashing functions from: + - [md5 module](lang-ref-ytt.md#md5) + - [sha256 module](lang-ref-ytt.md#sha256) +- Boolean expression operators and built-in functions, including: + - [`in`](https://github.com/google/starlark-go/blob/master/doc/spec.md#membership-tests) (aka "membership test") + - [`and` and `or`](https://github.com/google/starlark-go/blob/master/doc/spec.md#or-and-and) + - [`any()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#any) or [`all()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#all) + - [`hasattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#hasattr) + - [`len()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#len) + - [`type()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#type) +- [Language: String](lang-ref-string.md) functions + +--- +## Functions + +The `@ytt:overlay` module provides several functions that support overlay use. + +To use these functions, include the `@ytt:overlay` module: + +```python +#@ load("@ytt:overlay", "overlay") +``` + +The functions exported by this module are: +- [overlay.apply()](#overlayapply) +- [overlay.map_key()](#overlaymap_key) +- [overlay.index()](#overlayindex) +- [overlay.all()](#overlayall) +- [overlay.subset()](#overlaysubset) +- [overlay.and_op()](#overlayand_op) +- [overlay.or_op()](#overlayor_op) +- [overlay.not_op()](#overlaynot_op) + +### overlay.apply() + +Executes the supplied overlays on top of the given structure. + +```python +overlay.apply(left, right1[, rightN...]) +``` + +- `left` ([`yamlfragment`](lang-ref-yaml-fragment.md)) — the target of the overlays +- `right1` ([`yamlfragment`](lang-ref-yaml-fragment.md) annotated with [`@overlay/(action)`](#action-annotations)) — the (first) overlay to apply on `left`. +- `rightN` ([`yamlfragment`](lang-ref-yaml-fragment.md) annotated with [`@overlay/(action)`](#action-annotations)) — the Nth overlay to apply on the result so far (which reflects the changes made by prior overlays) + +**Notes:** +- For details on how to use `apply()`, see [Programmatic access](#programmatic-access). + +**Examples:** + +```python +overlay.apply(left(), right()) +overlay.apply(left(), one(), two()) +``` + +See also: [Overlay example](/ytt/#example:example-overlay) in the ytt Playground. + +### overlay.map_key() + +An [Overlay matcher function](#overlaymatch) that matches when the collection (i.e. Map or Array) in the "left" contains a map item with the key of `name` and value equal to the corresponding map item from the "right." + +```python +overlay.map_key(name) +``` +- `name` (`String`) — the key of the contained map item on which to match + +**Note:** this matcher requires that _all_ items in the target collection have a map item with the key `name`; if this requirement cannot be guaranteed, consider using [`overlay.subset()`](#overlaysubset), instead. + +**Examples:** + +_Example 1: Over an Array_ + +With "left" similar to: +```yaml +clients: +- id: 1 +- id: 2 +``` +the following matches on the second array item: +```yaml +clients: +#@overlay/match by=overlay.map_key("id") +- id: 2 +``` + +_Example 2: Over a Map_ + +(as of v0.26.0+) + +With "left" similar to: +```yaml +clients: + clientA: + id: 1 + clientB: + id: 2 +``` +the following matches on the second map item: +```yaml +--- +clients: + #@overlay/match by=overlay.map_key("id") + _: + id: 2 +``` +(note: the key name `_` is arbitrary and ignored). + +### overlay.index() + +An [Overlay matcher function](#overlaymatch) that matches the array item at the given index + +```python +overlay.index(i) +``` +- `i` (`Int`) — the ordinal of the item in the array on the "left" to match (zero-based index) + +**Example:** + +```yaml +#@overlay/match by=overlay.index(0) +- item10 +``` + +### overlay.all() + +An [Overlay matcher function](#overlaymatch) that matches all contained nodes from the "left", unconditionally. + +```python +overlay.all() +``` +_(this function has no parameters.)_ + +**Examples:** + +_Example 1: Documents_ + +Matches each and every document: +```yaml +#@overlay/match by=overlay.all +--- +metadata: + annotations: ... +``` + +_Example 2: Array Items_ + +Matches each and every item in the array contained in `items` on the "left": +```yaml +items: +#@overlay/match by=overlay.all +- item10 +``` + +_Example 3: Map Items_ + +(as of v0.26.0+) + +Matches each and every item in the map contained in `items` on the "left": +```yaml +items: + #@overlay/match by=overlay.all + _: + name: item10 +``` +(note: the key name `_` is arbitrary and ignored) + +### overlay.subset() + +An [Overlay matcher function](#overlaymatch) that matches when the "left" node's structure and value equals the given `target`. + +```python +overlay.subset(target) +``` +- `target` (`any`) — value that the "left" node must equal. + +**Examples** + +_Example 1: Scalar_ + +To match, scalar values must be equal. + +```yaml +#@overlay/match by=overlay.subset(1) +#@overlay/match by=overlay.subset("Entire string must match") +#@overlay/match by=overlay.subset(True) +``` +(if a partial match is required, consider writing a [Custom Overlay Matcher function](#custom-overlay-matcher-functions)) + +_Example 2: Dict (aka "map")_ + +To match, dictionary literals must match the structure and value of `left`. + +```yaml +#@overlay/match by=overlay.subset({"kind": "Deployment"}) +#@overlay/match by=overlay.subset({"metadata": {"name": "istio-system"}}) +``` + +_Example 3: YAML Fragment_ + +To match, [`yamlfragment`](lang-ref-yaml-fragment.md)'s must match structure and value of `left`. + +```yaml +#@ def resource(kind, name): +kind: #@ kind +metadata: + name: #@ name +#@ end + +#@overlay/match by=overlay.subset(resource("Deployment", "istio-system")) +``` + +### overlay.and_op() + +(as of v0.26.0+) + +An [Overlay matcher function](#overlaymatch) that matches when all given matchers return `True`. + +```python +overlay.and_op(matcher1, matcher2, ...) +``` +- `matcher1`, `matcher2`, ... — one or more other [Overlay matcher function](#overlaymatch)s. + +**Examples:** + +```yaml +#@ not_sa = overlay.not_op(overlay.subset({"kind": "ServiceAccount"})) +#@ inside_ns = overlay.subset({"metadata": {"namespace": "some-ns"}}) + +#@overlay/match by=overlay.and_op(not_sa, inside_ns),expects="1+" +``` + +### overlay.or_op() + +(as of v0.26.0+) + +An [Overlay matcher function](#overlaymatch) that matches when at least one of the given matchers return `True`. + +```python +overlay.or_op(matcher1, matcher2, ...) +``` +- `matcher1`, `matcher2`, ... — one or more other [Overlay matcher function](#overlaymatch)s. + +**Examples:** + +```yaml +#@ config_maps = overlay.subset({"kind": "ConfigMap"}) +#@ secrets = overlay.subset({"kind": "Secret"}) + +#@overlay/match by=overlay.or_op(config_maps, secrets) +``` + +### overlay.not_op() + +(as of v0.26.0+) + +An [Overlay matcher function](#overlaymatch) that matches when the given matcher does not. + +```pythons +not_op(matcher) +``` +- `matcher` — another [Overlay matcher function](#overlaymatch). + +```yaml +#@overlay/match by=overlay.not_op(overlay.subset({"metadata": {"namespace": "app"}})) +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-schema.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-schema.md new file mode 100644 index 000000000..42160d800 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-schema.md @@ -0,0 +1,701 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-schema] +title: Data Values Schema Reference +--- + +## Overview + +This reference covers details of Data Values Schema: supported types and annotations. + +For an introduction of Data Values, see [Using Data Values](how-to-use-data-values.md). \ +For details on writing Schema, see [Writing Schema](how-to-write-schema.md). + +## The Schema Document + +Schema is written in YAML. + +```yaml +#@data/values-schema +--- +... +``` +where: +- the document must be annotated as `@data/values-schema` +- each item in the document [declares a Data Value](#data-value-declarations) (i.e. an item in the [data.values](lang-ref-ytt.md#data) struct). +- (optionally) types and default values can be explicitly specified through [annotations](#annotations). +- a file containing a Schema document must not contain any other kind of document. + +### Multiple Schema Documents + +In some cases, it is useful to separate Schema into multiple YAML documents (typically in separate files). + +When doing so, it becomes relevant to know that Schema Documents are [`ytt` Overlays](ytt-overlays.md): +- they merge in [the same order as overlays](lang-ref-ytt-overlay.md#overlay-order), and +- one controls that merge via [overlay annotations](lang-ref-ytt-overlay.md#overlay-annotations). + +--- +## Data Value Declarations + +Each item in a Schema Document declares a Data Value. + +A Data Value declaration has three (3) parts: +- the **name** of the data value, +- its **default** value, and +- the **type** of the value. + +--- +### Names + +A Data Value is referred to by its name (aka "key" or "attribute"). \ +A Data Value name must be a string. + +When using multi-word names, it is recommended to employ snake-case (e.g. `database_connection`). This principally because: +- the underlying programming language in `ytt` — Starlark — is Pythonic in which identifiers are snake-cased, by convention +- as in most modern languages, the dash (i.e. `-`) is not allowed for identifier names in Starlark (allow characters are: Unicode letters, decimal digits, and underscores `_`, per the [Starlark spec](https://github.com/google/starlark-go/blob/master/doc/spec.md#lexical-elements)). + +Where disallowed characters in names cannot be avoided, references will need to employ either: +- the Starlark built-in [`getattr()`](https://github.com/google/starlark-go/blob/master/doc/spec.md#getattr) + ```python + secure: #@ getattr(data.values, "db-conn").secure + ``` +- _(as of v0.31.0)_ or [index notation](lang-ref-structs.md#attributes): + ```python + secure: #@ data.values["db-conn"].secure + ``` + +--- +### Default Values + +The default value for a Data Value is specified in schema, directly: + +- the default value for a **scalar** is the value given; +- the default value for a **map** are all of the items specified in the schema (with _their_ defaults, recursively); +- the default value for an **array** is an empty list (i.e. `[]`). + - the default value for _an item_ in an array are the contents of the item specified in schema (with _their_ defaults, recursively). + +Default values can be overridden using the [@schema/default](#schemadefault) annotation. + +#### Defaults for Scalars + +When a [scalar value](#types-of-scalars) is specified, the default is merely that value. + +For example, +```yaml +#@data/values-schema +--- +system_domain: "" + +load_balancer: + enabled: true + +databases: +- name: "" + adapter: postgresql +``` + +- `system_domain` is `""` (i.e. an empty string), by default. +- `load_balancer.enabled` is `true` by default. +- `databases[].adapter` is the string `"postgres"`, by default. + +#### Defaults for Mappings + +The set of items of a map are its default: any missing item will be automatically added with _its_ defaults. + +For example, +```yaml +#@data/values-schema +--- +load_balancer: + enabled: true + static_ip: "" +``` + +if `load_balancer` were omitted from supplied Data Values, entirely, it would default to: +- `load_balancer.enabled` is `true` +- `load_balancer.static_ip` is `""` + +if `load_balancer` is _partially_ specified... + +```yaml +#@data/values +--- +load_balancer: + static_ip: 10.0.101.1 +``` + +the missing item (here, `enabled`) is defaulted: +- `load_balancer.static_ip` is `"10.0.101.1"` +- `load_balancer.enabled` is `true` + + +#### Defaults for Arrays + +The default value for all arrays is, by default, an empty array. + +This is different from all other types where the default value is literally what is specified in schema. For arrays, it is `[]` (i.e. an empty array). + +This means that the value given for the element is _only_ used to infer the type of the array's _elements_. + +```yaml +#@data/values-schema +--- +app_domains: +- "" + +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + user: admin + secretRef: + name: "" +``` + +- `app_domains` is `[]` by default. Each item in the array must be a **string**. +- `databases` is `[]` by default. Each item in the array must be a **map**. When an item is added to the array: + - its key must be one of those named in this schema: `name`, `adapter`, `host`, etc. + - if it lacks any of the six keys, they will be added with their default values. + +The default value of an array _itself_ can be overridden using the [@schema/default](#schemadefault) annotation. + +--- +### Types + +`ytt` Schema can infer the type of the Data Value from the following... + +#### Types of Scalars + +`ytt` recognizes the following scalar types: + +- **strings** — e.g. `""`, `"ConfigMap"`, `"0xbeadcafe"` +- **integers** — e.g. `42` +- **booleans** — `true`, `false` (and when not strict, `yes`, `no`, `Y`, `N`, etc.) +- **floats** — e.g. `0.4` + +#### Types of Maps + +A map is a collection of map items, each a key/value pair. + +The schema of a map is inferred from that collection. Each item declares a nested Data Value of the type inferred from the given map item's value. + +For example, +```yaml +load_balancer: + enabled: true + static_ip: "" +``` + +where: +- `load_balancer` has a type of a map that has two items: + - one item has a key `enabled` whose type is a **boolean**. + - the other item has a key of `static_ip` and is a **string**. + +#### Types of Arrays + +An array is a sequence of array items. + +The schema of an array must contain _exactly_ one (1) item. The type inferred from that item becomes the type of _all_ items in that array. That is, arrays in `ytt` are homogenous. + +For example, +```yaml +app_domains: +- "" +``` +where: +- `app_domains` has a type of an array. Each element in that array will be a **string**. +- note that the default value for `app_domains` is an empty list as explained in [Defaults for Arrays](#defaults-for-arrays), above. + +#### `null` Type + +The `null` value means the absence of a value. + +In `ytt` schema, a default value is not permitted to be `null` (with one exception described in [`any` Type](#any-type), below). This is because no useful type can be inferred from the value `null`. + +Instead, one provides a non-null default value and annotates the Data Value as "nullable". + +This results in a Data Value whose default value is `null`, but when set to a non-null value has an explicit type. See [`@schema/nullable`](#schemanullable) for details. + +#### `any` Type + +In certain cases, it may be necessary to relax all restrictions on the type or shape of a Data Value: + +- the Data Value is a pass-through, where template(s) using it merely insert its value, but care not about the actual contents; +- a heterogeneous array is required; +- there are multiple possible allowed types for a given Data Value. + +This is done by annotating the Data Value as having "any" type. See [`@schema/type`](#schematype) for details. + +--- +## Annotations + +`ytt` determines the type of each Data Value by _inferring_ it from the value specified in the schema file (as described in [Types](#types), above). Currently, there is no way to _explicitly_ set the type of a Data Value. + +Configuration Authors can explicit specify the type of a Data Value in two cases that are **not** inferrable: +- overriding the default value (via the [@schema/default](#schemadefault) annotation); +- also allowing null (via the [@schema/nullable](#schemanullable) annotation); +- allowing any type (via the [@schema/type](#schematype) annotation). + +### @schema/default + +Overrides the default value for the annotated node. + +```yaml +#@schema/default default_value +``` +- `default_value` — the value to set as the default for the annotated node. + - this value must be of the same type as the value given on the node. + +_(as of v0.38.0)_ + +_Example 1: Default value for an array of scalars_ + +```yaml +#@data/values-schema +--- +#@schema/default ["apps.example.com", "gateway.example.com"] +app_domains: +- "" +``` + +... yields the default: + +```yaml +app_domains: +- apps.example.com +- gateway.example.com +``` + +_Example 2: Default value for an array of maps_ + +When specifying values for an array of maps, it can quickly become unwieldy to keep on a single line. + +To handle these situations, enclose those values in a [Fragment function](lang-ref-yaml-fragment.md) and invoke that function as the value for `@schema/default`: + +```yaml +#! For best results, declare functions *before* the schema document. +#@ def default_dbs(): +- name: core + host: coredb + user: app1 +- name: audit + host: metrics.svc.local + user: observer +#@ end + +#@data/values-schema +--- +#@schema/default default_dbs() +databases: +- name: "" + adapter: postgresql + host: "" + port: 5432 + user: admin + secretRef: + name: "" +``` + +Yields the default: + +```yaml +databases: +- name: core + adapter: postgresql + host: coredb + port: 5432 + user: app1 + secretRef: + name: "" +- name: audit + adapter: postgresql + host: metrics.svc.local + port: 5432 + user: observer + secretRef: + name: "" +``` + +Note: as the comment in the example schema indicates, it is best to declare the function prior to starting the schema document itself (see https://github.com/carvel-dev/ytt/issues/526 for details). + +### @schema/nullable + +Extends the type of the Data Value to also allow `null` _and_ sets the default value to be `null`. + +```yaml +#@schema/nullable +``` + +**Unset value for strings** + +The preferred way to express "unset value" for a string-type is _not_ to mark it as "nullable" but to provide the empty value: `""`. Empty values in Starlark are falsey (details in the [Starlark Spec > Booleans](https://github.com/google/starlark-go/blob/master/doc/spec.md#booleans)). + +When empty string is a useful/valid value for a given Data Value, _then_ marking it as "nullable" is appropriate. In this case, one must take care to explicitly check if that Data Value is not [`None`](lang.md#types). + +_Example: Nullable map_ + +```yaml +#@data/values-schema +--- +#@schema/nullable +aws: + username: admin + password: "1234" + +name: "" +``` + +Without other Data Value settings, `aws` is `null` by default: +```yaml +aws: null +name: "" +``` + +However, if a Data Value is set: + +```bash +$ ytt ... --data-value aws.username=sa ... +``` + +That effectively sets `aws` to be non-null: `username` is set to the custom value and `password` is defaulted. + +```yaml +aws: + username: sa + password: "1234" + +name: "" +``` + +### @schema/type + +Explicitly configures the type of the annotated node. Currently, the only supported configuration is whether to allow the "any" type or not. + +```yaml +#@schema/type any=True +``` + +where: +- `any` (`bool`) — whether or not any and all types are permitted on this node and its children. + +The annotated node and its nested children are not checked by schema, and has no schema defaulting behavior. +However, _(as of v0.39.0)_ any nested `@schema` annotation that alters type or value of a child would conflict with the fragment's "any" type, resulting in an error. +Otherwise, the annotated node and its children are simply passed-through as a data value. + +_Example: Using any=True to avoid schema restrictions on an array_ + +```yaml +#@data/values-schema +--- +#@schema/type any=True +app_domains: + - "carvel.dev" + - 8080 +``` + +_Example: Error case when setting schema within an any type fragment_ + +```yaml +#@data/values-schema +--- +#@schema/type any=True +app_domains: + #@schema/default "localhost" + #@schema/type any=False + - "carvel.dev" +``` +``` +ytt: Error: + Invalid schema + ============== + + Schema was specified within an "any type" fragment + schema.yml: + | + 5 | #@schema/default "localhost" + 6 | #@schema/type any=False + 7 | - "carvel.dev" + | + + = found: @schema/type, @schema/default annotation(s) + = expected: no '@schema/...' on nodes within a node annotated '@schema/type any=True' +``` + +### @schema/validation + +Attaches a validation to the type being declared by the annotated node. + +``` +@schema/validation rule0, rule1, ... [,] [,when=] +``` + +where: +- `ruleX` — any number of custom rules, each a 2-item tuple `(description, assertion)` + - `description` (`string`) — a description of what a valid value is. + - `assertion` (`function(value) : bool`) — returns `True` is the value is valid; returns `False` or `fail()`s if the value is invalid. + - `value` (`string` | `number` | `bool` | [`yamlfragment`](lang-ref-yaml-fragment.md)) — the value of the annotated node. + - the message given in the `fail()` will be incorporated in the violation message. +- `` — a combination of one or more built-in keywords that provide assertion functions for common scenarios. + - for a quick reference, see [Validations Cheat Sheet](quick-ref-validations.md). + - for the complete list, see [Named Validation Rules](#named-validation-rules), below. +- `when=` (`function(value[, context]) : bool`) — criteria for when the validation rules should be checked. + - `value` (`string` | `int` | `float` | `bool` | [`yamlfragment`](lang-ref-yaml-fragment.md)) — the value of the annotated node + - `context` (_optional_) — a struct with two attributes (both of the same _type_ as `value`). + - `parent` — the node directly containing the annotated node (in other words, its parent) + - `root` — the document in which the annotated node is contained (that is, the root of document resulting from the merge of all data values) + +When present, the predicate given for `when=` is run. If it evaluates to `True`, the validation is run. For guidance on writing these conditions, see [How To Write Validations](how-to-write-validations.md#conditional-validations). + +For a quick reference of various rules and idioms, see [Schema Validation Cheat Sheet](schema-validations-cheat-sheet.md). + +Below: +- [Example 1: Using "named" rules](#example-1-using-named-rules) +- [Example 2: Using `when=`](#example-2-using-when) +- [Example 3: Accessing other data values within `when=`](#example-3-accessing-other-data-values-within-when) +- [Validation Rule Evaluation](#validation-rule-evaluation) +- [Named Validation Rules](#named-validation-rules) + + +#### Example 1: Using "named" rules + +```yaml +#@data/values-schema +--- +#@schema/validation min_len=1 +namespace: "" + +#@schema/validation min_len=1 +hostname: "" + +port: + #@schema/validation min=1, max=32767 + https: 443 + +#@schema/validation one_of=["debug", "info", "warning", "error", "fatal"] +logLevel: info + +#@schema/nullable +tlsCertificate: + #@schema/validation min_len=1 + tls.crt: "" + #@schema/validation min_len=1 + tls.key: "" + #@schema/nullable + ca.crt: "" +``` +where: +- `namespace` and `hostname` are "required" — they will require overrides from the user to pass validation. + - see [How to Write Validations: "Required" Data Values](how-to-write-validations.md#required-data-values) +- `port.https` must be between 1 and 32767, inclusive. +- `logLevel` can _only_ be one of the values given +- `tlsCertificate` is optional, by default (and is `null`) + - however, if one or more of its values are set, _then_ both `tls.crt` and `tls.key` are "required". + - `ca.crt` is optional (and defaults to `null`) + +#### Example 2: Using `when=` + +```yaml +#@data/values-schema +--- +#@schema/validation ("have 1+ response type", lambda v: len(v["responseTypes"]) > 0), when=lambda v: v["enabled"] == True +oauth2: + enabled: true + responseTypes: + - "" +``` +where: +- validation on `oauth2` will only be run if `oauth2.enabled` is true. +- `v` is a [YAML Fragment](lang-ref-yaml-fragment.md) (hence the need for index notation to traverse the tree). + +#### Example 3: Accessing other data values within `when=` + +```yaml +#@data/values-schema +--- +credential: + useDefaultSecret: true + #@schema/nullable + #@schema/validation not_null=True, when=lambda _, ctx: ctx.parent["useDefaultSecret"] + secretContents: + cloud: "" +backupStorageLocation: + spec: + #@schema/nullable + #@schema/validation not_null=True, when=lambda _, ctx: not ctx.root["credential"]["useDefaultSecret"] + existingSecret: "" +#@schema/validation ("have 1+ response type", lambda v: len(v["responseTypes"]) > 0), when=lambda v: v["enabled"] == True +oauth2: + enabled: true + responseTypes: + - "" +``` +where: +- `secretContents` validation is dependent on its sibling `useDefaultSecret` value. + - the value of `when=` is a lambda expression making it possible to define a function value in-place. + - Lambda expressions start with the keyword `lambda`, followed by a parameter list, then a `:`, and a single expression that is the body of the function. For more detail see [Starlark Spec: lambda expression](https://github.com/google/starlark-go/blob/master/doc/spec.md#lambda-expressions). + - the `_` is an idiom for an ignored parameter. Here, the value of `secretContents` is not being used. + - the optional second parameter `ctx` where the `.parent` attribute refers to the value of `credential` +- `existingSecret` validation is dependent on `secretContents` which resides in a whole different subset of the data values. + - `ctx.root` refers to the top-most node of the document resulting from merging of all data values (as described in [How It Works: Calculating Data Values](how-it-works.md#step-1-calculate-data-values)). + +#### Validation Rule Evaluation + +When a validation runs, _all_ rules are evaluated. Rules are evaluated in the order they appear in the annotation (left-to-right). + +The one exception is the `not_null=` rule: +- when present, this rule is evaluated _first_ — regardless of its position on the annotation. +- if this rule is not satisfied, no other rules are evaluated. + +`not_null=` behaves this way so that all other rules can assume they are validating a non-null value. This simplifies all other rules as they need not perform a null-check. + +Validity: +- if _all_ rules pass (i.e. returns `True`), then the value is valid. +- if _any_ rule returns `False` or `fail()`'s, then the value is invalid. + + +#### Named Validation Rules + +There are seven (7) built-in (so-called "named") rules: + +- [`max=`](#max) — the node's value must be <= the maximum given +- [`max_len=`](#max_len) — the length of node's value must be <= the maximum given +- [`min=`](#min) — the node's value must be >= the minimum given +- [`min_len=`](#min_len) — the length of node's value must be >= the minimum given +- [`not_null=`](#not_null) — the node's value must not be `null` +- [`one_not_null=`](#one_not_null) — _exactly_ one (1) item in the map is not `null`. +- [`one_of=`](#one_of) — the node's value must be one from the given set. + +Every named rule is also available as a function in the [`@ytt:assert` module](lang-ref-ytt-assert.md). Having a function that behaves identically to the named rule makes it easier to transition to more customized experience. + + +What follows is a reference for each named rule. + +--- + +##### max= + +``` +@schema/validation max=maximum +``` + +Where: +- `maximum` (`int` | `float` | `string` | `bool` | `list` ) — the largest allowable valid value (inclusive); see [Starlark: Comparisons](https://github.com/google/starlark-go/blob/master/doc/spec.md#comparisons) for how different values compare. + +This keyword is equivalent to: + +``` +@schema/validation ("a value less than or equal to", lambda v: v <= maximum) +``` + +##### max_len= + +``` +@schema/validation max_len=maximum +``` + +Where: +- `maximum` (`int`) — the longest allowable length (inclusive) + +This keyword is equivalent to: + +``` +@schema/validation ("length less than or equal to", lambda v: len(v) <= maximum) +``` + +##### min= + +``` +@schema/validation min=minimum +``` + +Where: +- `minimum` (`int` | `float` | `string` | `bool` | `list` ) — the smallest allowable value (inclusive); see [Starlark: Comparisons](https://github.com/google/starlark-go/blob/master/doc/spec.md#comparisons) for how different values compare. + +This keyword is equivalent to: + +``` +@schema/validation ("a value greater than or equal to", lambda v: v >= minimum) +``` + +##### min_len= + +``` +@schema/validation min_len=(minimum) +``` + +Where: +- `minimum` (`int`) — the shortest allowable length (inclusive) + +This keyword is equivalent to: + +``` +@schema/validation ("length greater than or equal to", lambda v: len(v) >= minimum) +``` + +##### not_null= + +``` +@schema/validation not_null=(nullable) +``` + +Where: +- `nullable` (`bool`) — whether `null` is a valid value. + +When present, this rule is checked before any other; this allows other rules (including custom rules) to assume the value is not null. + + +This keyword is equivalent to: + +``` +@schema/validation ("not null", lambda v: v != None) +``` + +##### one_not_null= + +Requires a map to contain exactly one (1) item that has a non-null value. + +``` +@schema/validation one_not_null=([key0, key1,...] | True) +``` + +Where: +- `keyX` (`string`) — keys within the annotated map to check for null/non-null; other map items are ignored. +- `True` — check all contained map items for null/non-null. + +Each item that is referenced by this rule is almost always annotated as nullable: + +```yaml +#@schema/validation one_not_null=["item1", "item2", "item3"] +map: + #@schema/nullable + item1: "" + #@schema/nullable + item2: "" + #@schema/nullable + item3: "" + otherConfig: true +``` + +This keyword is equivalent to: + +``` +@schema/validation ("exactly one child not null", lambda v: assert.one_not_null([key0, key1,...])) +``` +(see also [@ytt:assert.one_not_null()](lang-ref-ytt-assert.md#assertone_not_null)) + +##### one_of= + +Requires value to be exactly one of the given enumeration. + +``` +@schema/validation one_of=[val0, val1, ...] +``` + +Where: +- `[val1, val2, ...]` (list/tuple of any type) — the exhaustive set of valid values. + +This keyword is equivalent to: + +``` +@schema/validation ("one of [val0, val1, ...]", lambda v: v in [val0, val1, ...]) +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-struct.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-struct.md new file mode 100644 index 000000000..a85bc898e --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-struct.md @@ -0,0 +1,203 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-struct] +title: Struct module +--- + +## Overview + +The `@ytt:struct` module provides functions for constructing and deconstructing [`struct`](lang-ref-structs.md) values. + +To use these functions, include the `@ytt:struct` module: + +```python +load("@ytt:struct", "struct") +``` + +--- +## struct.decode() + +Deconstructs a given value into plain/Starlark values, recursively. + +```python +struct.decode(struct_val) +``` + +- `struct_val` ([`struct`](lang-ref-structs.md)) — the value to decompose. + - `struct` values are converted into `dict` values where each attribute in the `struct` becomes a + key on the `dict`. + - if the value of an attribute is a `struct`, it is likewise converted to a `dict`. + - all other values are copied, as is. + +### Example + +```python +load("@ytt:struct", "struct") + +foo = struct.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +bar = struct.decode(foo) + +bar["a"] # <== [1, 2, 3, {"c": 456}] +``` + +--- +## struct.encode() + +Makes a `struct` of a given value, recursively. + +```python +struct.encode(value) +``` + +- `value` ([`dict`](lang-ref-dict.md) | [`list`](lang-ref-list.md) | scalar) — the value to encode. + - `dict` values are converted into `struct`s where each key in the `dict` becomes an attribute on the `struct`. + Keys of the items in `dict` values must be strings. + - if a `dict` or `list` contains a value that is a `dict`, it is likewise converted to a `struct`. + +Notes: + +- `encode()` cannot encode functions nor [YAML Fragments](lang-ref-yaml-fragment.md). If you wish to make a struct that + contains attributes that hold these types, consider [`make()`](#structmake). + +### Example: Data structure from a dictionary + +```python +load("@ytt:struct", "struct") + +d = struct.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +d.a # <== [1, 2, 3, c: (struct...)] +d.a[3].c # <== 456 +bar["b"] # <== "str" +``` + +## struct.make() + +Instantiates a `struct` based on the key/value pairs provided. + +```python +struct.make(key1=value1, key2=value2, ...) +``` + +- `keyN` (keyword argument name) — becomes the name of the Nth attribute in the constructed + `struct`. +- `=valueN`(`any`) — becomes the value of the Nth attribute in the constructed `struct`. + +Notes: + +- `make()` does not modify `values` in any way (e.g. if `valueN` is a dictionary, it is + _not_ converted into a `struct`). To recursively build a hierarchy of `struct`s from `dict`, + `list`, and scalars, see [`struct.encode()`](#structencode). + +### Example 1: Scalar values + +For visually pleasing collections of fields +```python +load("@ytt:struct", "struct") + +consts = struct.make(version="0.39.0", service_name="prometheus") + +consts.version # <== "0.39.0" +consts.service_name # <== "prometheus" +``` + +### Example 2: Data structures + +Dictionaries values remain instances of `dict`. +```python +load("@ytt:struct", "struct") + +consts = struct.make(service={"version": "0.39.0", "name": "prometheus"}) + +consts.service["version"] # <== "0.39.0" +consts.service["name"] # <== "prometheus" +# const.service.version # Error! "dict has no .version field or method" +``` + +### Example 3: Nested structs + +Nested invocations of `make()` to retain dot expression access. +```python +load("@ytt:struct", "struct") + +consts = struct.make(service=struct.make(version="0.39.0", name="prometheus")) + +consts.service.version # <== "0.39.0" +consts.service.name # <== "prometheus" +``` +See also: [`struct.encode()`](#structencode) to convert all `dict` values to `struct`s, recursively. + +### Example 4: Collection of functions + +"Export" a set of functions from a library file. + +`urls.star` +```python +load("@ytt:struct", "struct") + +def _valid_port(port): +... +end + +def _url_encode(url): +... +end + +urls = struct.make(valid_port= _valid_port, encode= _url_encode, ...) +``` +```yaml +#@ load("urls.star", "urls") + +#@ if/end urls.valid_port(...): +encoded: #@ urls.encode("encode_url") +``` +--- +## struct.make_and_bind() + +Binds one or more function(s) to a `struct`, making them method(s) on that struct. +This allows `struct`s to carry both data and behavior related to that data. + +```python +struct.make_and_bind(receiver, method_name1=function1, ...) +``` + +- `receiver` ([`struct`](lang-ref-structs.md)) — "object" to attach the function(s) to. +- `method_nameN` (keyword argument name) — the name that callers will specify to invoke the method. +- `functionN` ([`function`](lang-ref-def.md)) — the function value (either the name of a `function` or a `lambda` expression) + that will be bound to `receiver` by the name `method_nameN`. + - the first parameter of `functionN` is `receiver`, implicitly. + - the remaining parameters of `functionN` become the parameters of `receiver.method_nameN()` + +Notes: + +- Binding is useful for cases where a commonly desired value is a calculation of two or more + values on `receiver`. + +### Example 1: Binding a function value + +```python +load("@ytt:struct", "struct") + +conn = struct.make(hostname="svc.example.com", default_port=1022, protocol="https") + +def _url(self, port=None): + port = port or self.default_port + return "{}://{}:{}".format(self.protocol, self.hostname, port) +end + +conn = struct.make_and_bind(conn, url=_url) + +conn.url() # ==> https://svc.example.com:1022 +conn.url(8080) # ==> https://svc.example.com:8080 +``` + +### Example 2: Binding a lambda expression + +```python +load("@ytt:struct", "struct") + +_conn_data = struct.make(hostname="svc.example.com", default_port=1022, protocol="https") + +conn = struct.make_and_bind(_conn_data, url=lambda self, port=None: "{}://{}:{}".format(self.protocol, self.hostname, port or self.default_port)) + +conn.url() # ==> https://svc.example.com:1022 +conn.url(8080) # ==> https://svc.example.com:8080 +``` diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-template.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-template.md new file mode 100644 index 000000000..b813ee923 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-template.md @@ -0,0 +1,71 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-template] +title: Template Module +--- + +## `@template` Functions + +The `@ytt:template` module provides a function that can be used to update templates. + +To use these functions, include the `@ytt:template` module: + +```python +#@ load("@ytt:template", "template") +``` + +The functions exported by this module are: +- [template.replace()](#templatereplace) + +### template.replace() +Replaces the existing yaml node with the yaml node(s) provided or returned from a function call, of the same type. +Underscore (`_`) is the conventional replacement key, though any key can be used. + +``` +template.replace(node) +``` + +* `node` ([`yamlfragment`](lang-ref-yaml-fragment.md)) — yaml fragment that will replace the existing node + +**Examples:** + +##### Add a new item to the `labels` mapping +```yaml +#@ load("@ytt:template", "template") + +labels: + another-label: true + _: #@ template.replace({"key2": "value2"}) +``` +results in: +```yaml +labels: + another-label: true + key2: value2 +``` + +Notice that the argument to the function entirely replaces the `_` map item. + +##### Use a function instead of providing the item(s) inline +```yaml +#@ load("@ytt:template", "template") + +#@ def my_labels(): +#@ return {"key2": "value2"} +#@ end + +labels: + another-label: true + key-will-disappear: #@ template.replace(my_labels()) +``` +results in: +```yaml +labels: + another-label: true + key2: value2 +``` + +Notice that the argument to the `replace` function entirely replaces the `key-will-disappear` map item. + +So, regardless of the node's key or value, `template.replace` will overwrite it with the argument provided. + +See also: [Replace example](/ytt/#example:example-replace) in the ytt Playground. diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt-version.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-version.md new file mode 100644 index 000000000..0b70659f2 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt-version.md @@ -0,0 +1,25 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt-version] +title: Version module +--- + +Available in v0.26.0+ + +Version module provides a way to assert on minimum ytt binary version in your configuration. It could be placed into a conditional or just at the top level within `*.star`, `*.yml` or any other template file. + +Example configuration directory may look like this: + +- `config/` + - `0-min-version.star`: contents below + - `deployment.yml` + - `other.yml` + +```python +# filename starts with '0-' to make sure this file gets +# processed first, consequently forcing version check run first +load("@ytt:version", "version") + +version.require_at_least("0.26.0") +``` + +Note that ytt sorts files alphanumerically and executes templates in order. Most of the time it's best to check version first before processing other templates, hence, in the above example we've named file `0-min-version.star` so that it's first alphanumerically. diff --git a/site/content/ytt/docs/v0.50.x/lang-ref-ytt.md b/site/content/ytt/docs/v0.50.x/lang-ref-ytt.md new file mode 100644 index 000000000..fb8777d63 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang-ref-ytt.md @@ -0,0 +1,272 @@ +--- +aliases: [/ytt/docs/latest/lang-ref-ytt] +title: Built-in ytt Library +--- + +## General modules + +### assert + +See [@ytt:assert module docs](lang-ref-ytt-assert.md). + +### data + +See [Data Values](ytt-data-values.md) reference for more details + +```python +load("@ytt:data", "data") + +data.values # struct that has input values + +# relative to current package +data.list() # ["template.yml", "data/data.txt"] +data.read("data/data.txt") # "data-txt contents" + +# relative to library root (available in v0.27.1+) +data.list("/") # list files +data.read("/data/data.txt") # read file +``` + +### ip + +Parse and inspect Internet Protocol values. + +(available in v0.37.0+) + +```python +load("@ytt:ip", "ip") + +# Parse IP addresses... +addr = ip.parse_addr("192.0.2.1") +addr.is_ipv4() # True +addr.is_ipv6() # False +addr.string() # "192.0.2.1" + +addr = ip.parse_addr("2001:db8::1") +addr.is_ipv4() # False +addr.is_ipv6() # True +addr.string() # "2001:db8::1" + +# Parse CIDR notation into an IP Address and IP Network... +addr, net = ip.parse_cidr("192.0.2.1/24") +addr.string() # "192.0.2.1" +addr.is_ipv4() # True +addr.is_ipv6() # False +net.string() # "192.0.2.0/24" +net.addr().string() # "192.0.2.1" +net.addr().is_ipv4() # True +net.addr().is_ipv6() # False + +addr, net = ip.parse_cidr("2001:db8::1/96") +addr.string() # "2001:db8::1" +addr.is_ipv4() # False +addr.is_ipv6() # True +net.string() # "2001:db8::/96" +net.addr().string() # "2001:db8::" +net.addr().is_ipv4() # False +net.addr().is_ipv6() # True +``` + +### math + +Module math is a Starlark module of math-related functions and constants. + +> ⚠️ **Non-Deterministic Results** \ +> The functions in this module do not guarantee bit-identical results across CPU architectures. +> Using one or more of these functions may produce different output on different machines. + + +The module defines the following functions: + +_(All functions accept both int and float values as arguments.)_ + +```python +math.ceil(x) # the ceiling of x, the smallest integer greater than or equal to x. +math.copysign(x, y) # a value with the magnitude of x and the sign of y. +math.fabs(x) # the absolute value of x as float. +math.floor(x) # the floor of x, the largest integer less than or equal to x. +math.mod(x, y) # the floating-point remainder of x/y. The magnitude of the result is less than y and its sign agrees with that of x. +math.pow(x, y) # x**y, the base-x exponential of y. +math.remainder(x, y) # the IEEE 754 floating-point remainder of x/y. +math.round(x) # the nearest integer, rounding half away from zero. + +math.exp(x) # e raised to the power x, where e = 2.718281… is the base of natural logarithms. +math.sqrt(x) # the square root of x. + +math.acos(x) # the arc cosine of x, in radians. +math.asin(x) # the arc sine of x, in radians. +math.atan(x) # the arc tangent of x, in radians. +math.atan2(y, x) # atan(y / x), in radians. + # The result is between -pi and pi. + # The vector in the plane from the origin to point (x, y) makes this angle with the positive X axis. + # The point of atan2() is that the signs of both inputs are known to it, so it can compute the correct + # quadrant for the angle. + # For example, atan(1) and atan2(1, 1) are both pi/4, but atan2(-1, -1) is -3*pi/4. + +math.cos(x) # the cosine of x, in radians. +math.hypot(x, y) # the Euclidean norm, sqrt(x*x + y*y). This is the length of the vector from the origin to point (x, y). +math.sin(x) # the sine of x, in radians. +math.tan(x) # the tangent of x, in radians. + +math.degrees(x) # Converts angle x from radians to degrees. +math.radians(x) # Converts angle x from degrees to radians. + +math.acosh(x) # the inverse hyperbolic cosine of x. +math.asinh(x) # the inverse hyperbolic sine of x. +math.atanh(x) # the inverse hyperbolic tangent of x. +math.cosh(x) # the hyperbolic cosine of x. +math.sinh(x) # the hyperbolic sine of x. +math.tanh(x) # the hyperbolic tangent of x. + +math.log(x, base) # the logarithm of x in the given base, or natural logarithm by default. + +math.gamma(x) # the Gamma function of x. + +math.e # The base of natural logarithms, approximately 2.71828. +math.pi # The ratio of a circle's circumference to its diameter, approximately 3.14159. +``` + +### regexp + +```python +load("@ytt:regexp", "regexp") + +regexp.match("[a-z]+[0-9]+", "__hello123__") # True + +regexp.replace("[a-z]+[0-9]+", "__hello123__", "foo") # __foo__ +regexp.replace("(?i)[a-z]+[0-9]+", "__hello123__HI456__", "bye") # __bye__bye__ +regexp.replace("([a-z]+)[0-9]+", "__hello123__bye123__", "$1") # __hello__bye__ +regexp.replace("[a-z]+[0-9]+", "__hello123__", lambda s: str(len(s))) # __8__ + +# example of passing the "dot matches newline" flag and using replace to extract a single match from a multiline input string +input_str = "\\ multline string\n\nconst (\n\t// Value is what we want to scrape\n\tValue = 12\n)\n\nfunc main() {..." +regexp.replace("(?s).*Value = ([0-9]+).*", input_str, "$1") # 12 +``` + +See the [RE2 docs](https://github.com/google/re2/wiki/Syntax) for more on regex syntax. Note that flags such as multiline mode are passed in the pattern string as in the [golang regexp library](https://pkg.go.dev/regexp/syntax). + +When calling `replace` you can pass either a string or a lambda function as the third parameter. When given a string, `$` symbols are expanded, so that `$1` expands to the first submatch. When given a lambda function, the match is directly replaced by the result of the function. + +While `match` and `replace` are currently the only regexp verbs supported, it is possible to mimic `find` by using `replace` to replace all its input with a capture group (see example above). + +### struct + +See [@ytt:struct module docs](lang-ref-ytt-struct.md). + +### url + +```python +load("@ytt:url", "url") + +url.path_segment_encode("part part") # "part%20part" +url.path_segment_decode("part%20part") # "part part" + +url.query_param_value_encode("part part") # "part+part" +url.query_param_value_decode("part+part") # "part part" + +url.query_params_encode({"x":["1"],"y":["2","3"],"z":[""]}) # "x=1&y=2&y=3&z=" +url.query_params_decode("x=1&y=2&y=3;z") # (DEPRECATED) # {"x":["1"],"y":["2","3"],"z":[""]} +url.query_params_decode("x=1&y=2&y=3&z") # {"x":["1"],"y":["2","3"],"z":[""]} + +u = url.parse("http://alice:secret@example.com") +u.string() # "http://alice:secret@example.com" +u.user.name # "alice" +u.user.password # "secret" +u.user.string() # "alice:secret" +u.without_user().string() # "http://example.com" +``` + +As of v0.38.0, including semicolons in query strings is deprecated behavior. +Allowing semicolons in query strings can [lead to cache poisoning attacks](https://snyk.io/blog/cache-poisoning-in-popular-open-source-packages/). Authors should use ampersands (i.e. `&`) exclusively to separate parameters. + +### version + +`load("@ytt:version", "version")` (see [version module doc](lang-ref-ytt-version.md)) + +--- +## Hashing modules + +### md5 + +```python +load("@ytt:md5", "md5") + +md5.sum("data") # "8d777f385d3dfec8815d20f7496026dc" +``` + +### sha256 + +```python +load("@ytt:sha256", "sha256") + +sha256.sum("data") # "3a6eb0790f39ac87c94f3856b2dd2c5d110e6811602261a9a923d3bb23adc8b7" +``` + +--- +## Serialization modules + +### base64 + +```python +load("@ytt:base64", "base64") + +base64.encode("regular") # "cmVndWxhcg==" +base64.decode("cmVndWxhcg==") # "regular" +``` + +### json + +```python +load("@ytt:json", "json") + +json.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +json.encode({"a": [1,2,3,{"c":456}], "b": "str"}, indent=3) + +json.decode('{"a":[1,2,3,{"c":456}],"b":"str"}') +``` +As of v0.35.0, `json.encode()` with `indent` argument encodes result in multi-line string. + +### toml + +As of v0.38.0. + +```python +load("@ytt:toml", "toml") + +toml.encode({"a": [1,2,3,456], "b": "str"}) # 'a = [1, 2, 3, 456]\nb = "str"' +toml.encode({"metrics": {"address":"", "grpc_histogram": False}}, indent=4) + # '[metrics]\n address = ""\n grpc_histogram = false\n' + +toml.decode("[plugins]\n [plugins.cgroups]\n no_prometheus = false") + # {"plugins": {"cgroups": {"no_prometheus": False}}} +``` + + +### yaml + +```python +load("@ytt:yaml", "yaml") + +yaml.encode({"a": [1,2,3,{"c":456}], "b": "str"}) +yaml.decode('{"a":[1,2,3,{"c":456}],"b":"str"}') +``` + +--- +## Library module + +See [Library specific docs](lang-ref-ytt-library.md). + +--- +## Overlay module + +See [Overlay specific docs](lang-ref-ytt-overlay.md). + +--- +## Schema Module + +See [Schema specific docs](lang-ref-ytt-schema.md). + +--- +## Template module + +See [Template specific docs](lang-ref-ytt-template.md). diff --git a/site/content/ytt/docs/v0.50.x/lang.md b/site/content/ytt/docs/v0.50.x/lang.md new file mode 100644 index 000000000..d46f0a34c --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/lang.md @@ -0,0 +1,34 @@ +--- +aliases: [/ytt/docs/latest/lang] +title: Language +--- + +## Overview + +Templating language used in `ytt` is a slightly modified version of [Starlark](https://github.com/google/starlark-go/blob/master/doc/spec.md). Following modifications were made: + +- requires `end` keyword for block closing + - hence no longer whitespace sensitive (except new line breaks) +- does not allow use of `pass` keyword + +See [full Starlark specification](https://github.com/google/starlark-go/blob/master/doc/spec.md#contents) for detailed reference. + +## Types + +- NoneType: `None` (equivalent to null in other languages) +- Bool: `True` or `False` +- Integer: `1` +- Float: `1.1` +- [String](lang-ref-string.md): `"string"` +- [List](lang-ref-list.md): `[1, 2, {"a":3}]` +- Tuple: `(1, 2, "a")` +- [Dictionary](lang-ref-dict.md): `{"a": 1, "b": "b"}` +- [Struct](lang-ref-structs.md): `struct.make(field1=123, field2="val2")` +- [YAML fragment](lang-ref-yaml-fragment.md) +- [Annotation](lang-ref-annotation.md): `@name arg1,arg2,keyword_arg3=123` + +## Control flow + +- [If conditional](lang-ref-if.md) +- [For loop](lang-ref-for.md) +- [Function](lang-ref-def.md) diff --git a/site/content/ytt/docs/v0.50.x/outputs.md b/site/content/ytt/docs/v0.50.x/outputs.md new file mode 100644 index 000000000..6c9548312 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/outputs.md @@ -0,0 +1,17 @@ +--- +aliases: [/ytt/docs/latest/outputs] +title: Outputs +--- + +ytt supports three different output destinations: + +- stdout, which is default + - All YAML documents are combined into one document set. Non-YAML files are not printed anywhere. +- output files, controlled via `--output-files` flag (v0.28.0+) + - Output files will be added to given directory, preserving file names. + - Example: `ytt -f config.yml --output-files tmp/`. +- output directory, controlled via `--dangerous-emptied-output-directory` flag + - Given directory will be _emptied out_ beforehand and output files will be added preserving file names. + - Example: `ytt -f config.yml --dangerous-emptied-output-directory tmp/ytt/`. + +If you want to control which files are included in the output use `--file-mark 'something.yml:exclusive-for-output=true'` flag to mark one or more files. diff --git a/site/content/ytt/docs/v0.50.x/schema-validations-cheat-sheet.md b/site/content/ytt/docs/v0.50.x/schema-validations-cheat-sheet.md new file mode 100644 index 000000000..e20254858 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/schema-validations-cheat-sheet.md @@ -0,0 +1,214 @@ +--- +aliases: [/ytt/docs/latest/schema-validations-cheat-sheet] +title: "Schema Validations Cheat Sheet" +--- + +_(For a more detailed guide, see [Writing Schema Validations](how-to-write-validations.md).)_ + +{{< table class="cheat-sheet" >}} + +(th)**Use Case**(/th) +(th)**Syntax**(/th) +(tr) +(td) +##### Required string +_usage: [Using the empty/zero value](../how-to-write-validations#using-the-emptyzero-value)_\ +_reference: [`min_len=`](../lang-ref-ytt-schema#min_len)_ +(/td) +(td) +```yaml +#@schema/validation min_len=1 +username: "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Required integer +_usage: [Using the empty/zero value](../how-to-write-validations#using-the-emptyzero-value)_\ +_reference: [`min=`](../lang-ref-ytt-schema#min)_ +(/td) +(td) +```yaml +#@schema/validation min=1 +replicas: 0 +``` +(/td) +(/tr) + +(tr) +(td) +##### Required array +_usage: [Using the empty/zero value](../how-to-write-validations#using-the-emptyzero-value)_\ +_reference: [`min_len=`](../lang-ref-ytt-schema#min_len)_ +(/td) +(td) +```yaml +#@schema/validation min_len=1 +responseTypes: +- "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Required map +_usage: [mark as "nullable" and "not_null"](../how-to-write-validations#mark-as-nullable-and-not_null)_ \ +_reference: [`@schema/nullable`](../lang-ref-ytt-schema#schemanullable) and [`not_null=`](../lang-ref-ytt-schema#not_null)_ + +(/td) +(td) +```yaml +#@schema/nullable +#@schema/validation not_null=True +credential: + name: "" + cloud: "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Ensure string minimum length +_reference: [`min_len=`](../lang-ref-ytt-schema#min_len)_ +(/td) +(td) +```yaml +#@schema/validation min_len=8 +password: "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Ensure string exact length +(/td) +(td) +```yaml +#@schema/validation min_len=8, max_len=8 +password: "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Ensure a min value +(/td) +(td) +```yaml +#@schema/validation min=3 +replicas: 5 +``` +(/td) +(/tr) + +(tr) +(td) +##### Ensure a max value +(/td) +(td) +```yaml +#@schema/validation max=5 +replicas: 3 +``` +(/td) +(/tr) + +(tr) +(td) +##### Ensure a value between min and max +(/td) +(td) +```yaml +#@schema/validation min=1, max=65535 +port: 1024 +``` +(/td) +(/tr) + +(tr) +(td) +##### Enumeration +_usage: [enumerations](../how-to-write-validations#enumerations)_\ +_reference: [`one_of=`](../lang-ref-ytt-schema#one_of)_ +(/td) +(td) +```yaml +#@schema/validation one_of=["aws", "azure", "vsphere"] +provider: "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Exactly one is specified +_usage: [mutually exclusive config](../how-to-write-validations#mutually-exclusive-sections)_\ +_reference: [`one_not_null=`](../lang-ref-ytt-schema#one_not_null)_ +(/td) +(td) +```yaml +#@schema/validation one_not_null=["oidc", "ldap"] +config: + #@schema/nullable + oidc: + client_id: “” + #@schema/nullable + ldap: + host: “” +``` +(/td) +(/tr) + +(tr) +(td) +##### Conditionally run validations +_usage: [conditional validations](../how-to-write-validations#conditional-validations)_\ +_reference: [`@schema/validation ... when=`](../lang-ref-ytt-schema#schemavalidation)_ +(/td) +(td) +```yaml +#@data/values-schema +--- +service: + type: LoadBalancer + #@schema/validation min_len=1, when=lambda _, ctx: ctx.parent["type"] == "LoadBalancer" + name: "" +``` +(/td) +(/tr) + + +(tr) +(td) +##### Custom description of valid value +_usage: [writing custom rules](../how-to-write-validations#writing-custom-rules)_\ +_reference: [@ytt:assert](../lang-ref-ytt-assert)_ +(/td) +(td) +```yaml +#@ load("@ytt:assert", "assert") + +#@schema/validation ("a non-blank name", assert.min_len(1)) +username: "" +``` +(/td) +(/tr) + +(tr) +(td) +##### Disable validations flag +(/td) +(td) +```yaml +$ ytt ... --dangerous-data-values-disable-validation +``` +(/td) +(/tr) + +{{< /table >}} diff --git a/site/content/ytt/docs/v0.50.x/security.md b/site/content/ytt/docs/v0.50.x/security.md new file mode 100644 index 000000000..c8868cf88 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/security.md @@ -0,0 +1,34 @@ +--- +aliases: [/ytt/docs/latest/security] +title: Security +--- + +## Vulnerability Disclosure + +If you believe you have found a security issue in `ytt`, please privately and responsibly disclose it by following the directions in our [security policy](/shared/docs/latest/security-policy). + +## Attack Vectors + +This section is a work-in-progress... + +- malicious template input + - input tries to exhaust cpu/mem/disk resources + - A: how does it affect go-yaml? ... https://en.wikipedia.org/wiki/Billion_laughs_attack + - input tries to use YAML tagging to initialize custom objects (std yaml concern) + - A: TBD + +- malicious template code + - code tries to load file contents from sensitive locations + - A: templating is constrained to seeing only files explicitly specified by the user via -f flag, and does not follow symlinks. Unless user is tricked to provide sensitive files as input, template code is not able to access it. In other words, template runtime does not have facilities to access arbitrary filesystem locations. + - code tries to exfiltrate data over network + - A: template runtime does not have facilities to access network. + - code tries to exhaust cpu/mem/disk resources + - A: there are currently no resource constraints set by ytt itself for cpu/mem/disk. currently cpu can be pegged at 100% via an infinite loop. function recursion is also possible; however, it will be contstrained by Go stack space (and will exit the program). + - code tries to produce YAML that exhausts resources + - A: TBD + - meltdown/spectre style attacks + - A: TBD + +- CLI output directory + - user is tricked to set --output-files flag to a sensitive filesystem location + - A: template output is constrained to stdout or specified output directory via --output-files flag. if user is tricked to point --output-files flag to a sensitive filesystem location such as ~/.ssh/, attacker may be able to write templates (for example ~/.ssh/authorized_keys) that can be intepreted by the system as configuration/executable files. diff --git a/site/content/ytt/docs/v0.50.x/strict.md b/site/content/ytt/docs/v0.50.x/strict.md new file mode 100644 index 000000000..43a8b7e39 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/strict.md @@ -0,0 +1,48 @@ +--- +aliases: [/ytt/docs/latest/strict] +title: Strict YAML +--- + +## Overview + +ytt includes strict YAML subset mode that tries to remove any kind of ambiguity in user's intent when parsing YAML. + +Unlike full YAML, strict subset: + +- only supports specifying nulls as "" or `null` +- only supports specifying bools as `false` or `true` +- only support basic int and float declarations + - prefix, suffix, octal notation, etc are not supported +- requires strings with whitespace to be explicitly quoted +- requires strings with colon to be explicitly quoted +- requires strings with triple-dash (document start) to be explicitly quoted + +## Example + +Non-strict: + +```bash +$ echo 'key: yes' | ytt -f- +key: true +``` + +Strict: + +```bash +$ echo 'key: yes' | ytt -f- -s +Error: Unmarshaling YAML template 'stdin.yml': yaml: Strict parsing: Found 'yes' ambigious (could be !!str or !!bool) +``` + +To fix error, explicitly make it a string: + +```bash +$ echo 'key: "yes"' | ytt -f- -s +key: "yes" +``` + +or via YAML tag `!!str`: + +```bash +$ echo 'key: !!str yes' | ytt -f- -s +key: "yes" +``` diff --git a/site/content/ytt/docs/v0.50.x/yaml-primer.md b/site/content/ytt/docs/v0.50.x/yaml-primer.md new file mode 100644 index 000000000..cf95a8a6d --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/yaml-primer.md @@ -0,0 +1,228 @@ +--- +aliases: [/ytt/docs/latest/yaml-primer] +title: "YAML and Annotations" +--- + +## Overview + +Templates in `ytt` — rather than being YAML-like text files with templating injected _in_ — are well-formed YAML files with templating annotated _on_. + +It's reasonable to have expectations of how to write templates, especially if we have prior experience. But in the shift from a free-form text file to a structured YAML document set, some of those expectations are foiled. And when they are, it can be rather frustrating. + +If we take a few minutes to cover: + +- [what YAML is](#what-is-yaml) — its format and constituent parts; and +- [how to annotate YAML in `ytt`](#annotating-yaml), + +it can mitigate much of that frustration. + +## What is YAML? + +YAML is a tree of nodes. Each node is of one of these six types: + +- a **Document Set** (which is list of **Documents**) +- a **Map** that has **Map Items** +- an **Array** that has **Array Items** + +We'll start from the top of the tree... + +### Documents and Document Sets + +A file containing text in YAML format is parsed into a **Document Set**: a collection of **Document**s. + +![a file containing two YAML documents](/images/ytt/yaml-primer/two-docs.jpg) + +_Figure 1: A file, parsed into a **document set** (dotted blue) containing two **documents** (solid blue)._ + +Note: `---` marks the start of a new document. + +The corresponding tree of nodes looks something like this: + +![](/images/ytt/yaml-primer/two-docs-ast.jpg) + +_Figure 2: Corresponding tree: this document set has two documents._ + +A given document has exactly one (1) value. It is usually either: +- a **Map**, +- an **Array**, or +- a **Scalar** + +We'll look at each of these, in turn. Maps are most common... + +### Maps and Map Items + +A "map" is collection of key-value pairs. Each pair is referred to as a "map item". + +![](/images/ytt/yaml-primer/two-maps-and-items.jpg) + +_Figure 3: Each document has a **map** (dotted green) each containing two **map items** (solid green)._ + +The complete tree of nodes looks like this: + +![](/images/ytt/yaml-primer/two-maps-and-items-ast.jpg) + +_Figure 4: Corresponding tree: each document has a map for its value; each map has two items._ + +Let's zoom in on the first document, and explicitly reveal the contents of a map item: + +![](/images/ytt/yaml-primer/map-items-ast.jpg) + +_Figure 5: Each map item has a **key** and a **value**; here, both map items' key is a string and their value is an integer._ + +Like documents, a map item has exactly one (1) value. And just like documents, it's either: +- a **Map**, +- an **Array**, or +- a **Scalar** + +Let's see what it looks like for a **map item** to have a **map**, itself. + +![](/images/ytt/yaml-primer/map-item-has-map-ast.jpg) + +_Figure 6: The document's value is a map with two map items; the second map item has a key of "metadata" and a value that is another map._ + + +### Arrays and Array Items + +An "array" is a list of "array items" (zero-indexed). + +Like documents (and map items), an array item has exactly one (1) value. Once more, it's either: +- a **Map** +- an **Array**, or +- a **Scalar** + +![](/images/ytt/yaml-primer/map-item-has-array-ast.jpg) + +_Figure 7: The **map item** "foo" has an **array** (dotted gold) for a value; that array has three **array items** (solid gold)._ + +Let's see what it looks like when an **array item**'s value is a **map**... + +![](/images/ytt/yaml-primer/array-item-has-map-ast.jpg) + +_Figure 8: "foo" has an **array** with one **array item** whose value is a **map** containing two **map item**s: `name` and `factor`._ + +We've seen scalars in use above. Let's cover them explicitly. + +### Scalars + +A "scalar" is a fundamental value. YAML supports: +- integers (e.g. 13, 42, 137, 32767) +- floating point numbers (e.g. 1.0, 3.14, 137.03599913) +- strings (e.g. "", "fine structure", "$ecr3t") +- boolean values: `true` or `false` +- `null` (when a value is omitted, that implies `null`; `~` == `null`) + +### Summary + +In short: + +- a YAML file is parsed into a **Document Set** which is merely a list of **Document**s. +- a given **Document** has one _value_. +- Both **Map**s and **Array**s are nothing more than collections of items. + - a **Map Item** has a _key_ and a _value_ + - an **Array Item** has a _value_ +- a _value_ (whether held by a document, map item, or array item) is either a **Map**, an **Array**, or a **Scalar**. + +With this structure in mind, we can now look into how `ytt` annotates it. + +## Annotating YAML + +A `ytt` "annotation" is a YAML comment of a specific format; it attaches behavior to a node. + +For example: + +![](/images/ytt/yaml-primer/map-item-with-value.jpg) + +_Figure 9: A YAML file with a `ytt` annotation._ + +Can be understood to mean that the value of `foo` is the result of evaluating the expression `13 + 23 + 6`. + +The diagram reveals how the annotation is attached to the YAML tree: + +![](/images/ytt/yaml-primer/map-item-ann-with-value-ast.jpg) + +_Figure 10: The annotation (dotted black) is attached to the **map item** (solid green)._ + +This is a document, whose value is a map, that contains a single map item. To the right of the map item is an annotation that contains an expression that, when evaluated, becomes a _value_. The annotation is attached to the map item. + +### How an Annotation Finds its Node + +When attaching annotations, `ytt` follows these two rules (in order): +1. the annotation is attached to the value-holding node **on its left**, if there is one; otherwise, +2. the annotation is attached to the value-holding node **directly below** it. + +In the example above, the value-holding node to the left of the annotation is the map item. + +As noted in the summary [above](#summary), "value-holding" nodes are: +- **Document**s, +- **Map Item**s, and +- **Array Item**s. + +#### Example: Annotating Map Items + +Let's see these rules at play: + +![](/images/ytt/yaml-primer/overlay-ann-on-doc-and-map.jpg) + +_Figure 11: A templated document with three annotations._ + +We'll ignore _what_ these annotations mean, for now, and focus on _where_ they attach. There are three annotations in total. + +Visualizing the YAML tree can help: + +![](/images/ytt/yaml-primer/overlay-ann-on-doc-and-map-ast.jpg) + +_Figure 12: Each annotation attaches to the value-holding node to its left or bottom._ + +Taking each annotation in turn: +- `@overlay/match by=overlay.all`: + - is in the document set, but that is not a value-holding node + - has a **document** node just below it and so is attached to _that_ node. +- `@overlay/replace`: + - has no node to its left; + - has a **map** just below it but maps are _not_ value-holding; + - the next node is a **map item**, and so attaches to _that_ node. +- `@ 13 + 23 + 6`: + - has a **map item** to its left, and so attaches to _that_ node. + - there _is_ another **map item** below it (i.e. `bar: true`), but a home has already been found for this annotation. + +#### Example: Annotating an Array + +When an **array item** contains a **map**, it can be tricky to know which item a particular annotation attaches to. + +In this example, we reinforce the value of knowing the two rules that determine [how an annotation finds its node](#how-an-annotation-finds-its-node). + +![](/images/ytt/yaml-primer/overlay-ann-on-array-and-map.jpg) + +_Figure 13: An overlay ensuring each book has been marked as reviewed. The blank line after the array item is intentional._ + +There are three annotations in this example. The first one clearly annotates the document. But what about the next two? + +Visualizing the layer, it can start to become more clear... + +![](/images/ytt/yaml-primer/overlay-ann-on-array-and-map-ast.jpg) + +_Figure 14: The newline after the **array item** ensures the last annotation attaches to the **map item**._ + +Taking each annotation in turn: +- the first `@overlay/match by=overlay.all`: + - has a **document** node just below it and so is attached to _that_ node. +- the second `@overlay/match by=overlay.all`: + - has a **map item** (`books:`), but that is _above_ this annotation; + - `books:` contains an **array**, but arrays are _not_ value-holding nodes; + - that array's **array item** (denoted by `-`) is just below the annotation and so is attached to _that_ node. +- finally, `@overlay/match missing_ok=True`: + - if this _had_ been the same line as the **array item**, it _would_ have also attached to the same node as the previous annotation (but it is not, and is it won't); + - the next node is a **map**, but again, maps are _not_ value-holding; + - the next node is a **map item** (`reviewed:`), so the annotation attaches to _that_ node. + + +## Further Exploration + +While we've thoroughly covered the fundamentals here, these concepts only become real with use: + +- put your understanding to the test by picking an example from [the Playground](https://carvel.dev/ytt/#example:example-demo) and tweaking it in various ways to see what _actually_ happens. +- take a simple example and run it through `ytt --debug` and study the `### ast` section of the output. Note: + - in this output, annotations are referred to as `meta`. + - "AST" is short for "Abstract Syntax Tree" which is the technical name for the trees we've been visualizing in this guide. + - we recommend making small changes and study how it modifies the "AST" +- if you'd like to obsess over YAML itself, look no further than the [YAML spec](https://yaml.org/spec/1.2/spec.html). diff --git a/site/content/ytt/docs/v0.50.x/ytt-data-values.md b/site/content/ytt/docs/v0.50.x/ytt-data-values.md new file mode 100644 index 000000000..123d1e62f --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/ytt-data-values.md @@ -0,0 +1,446 @@ +--- +aliases: [/ytt/docs/latest/ytt-data-values] +title: Data Values +--- + +## Overview + +A `ytt` run can be configured by supplying custom Data Values. + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +## Declaring Data Values + +Typically, Data Values are declared in a schema file. See the [Using Data Values](how-to-use-data-values.md) guide for more details. + +Note: `ytt` continues to support declaring Data Values without schema for backwards-compatibility. However, due to the significantly improved support for catching configuration errors that schema brings, it is the recommended method for doing so. + +## Configuring Data Values + +Data Values can be configured in one of two ways: + +- on the command-line via the family of [command-line `--data-value...` flags](#configuring-data-values-via-command-line-flags), +- in a ["Data Values Overlay" document](#configuring-data-values-via-data-values-overlays) and included via the `--file` flag, + +### Configuring Data Values via command line flags + +The `--data-value...` family of command-line flags provides a means of configuring Data Values from: + +- the command-line, multiple values: + - [`--data-values-file`](#--data-values-file) — multiple values from a file, directory, URL, or standard in. +- the command-line, directly, one at a time: + - [`--data-value`](#--data-value) — single value as a string. + - [`--data-value-yaml`](#--data-value-yaml) — single value decoded a YAML. + - [`--data-value-file`](#--data-value-file) — a single value to the contents of a file, URL, or standard in. +- OS environment variables: + - [`--data-values-env`](#--data-values-env) — all OS variables with a named prefix, all as strings. + - [`--data-values-env-yaml`](#--data-values-env-yaml) — all OS variables with a named prefix, decoded as YAML. + +**Quick Example** + +```bash +export STR_VALS_key6=true +export YAML_VALS_key7=true + +$ cat dev/values.yml +key1: values.yml-key1 +key2: + original: from values.yml + +$ ytt \ + --data-value key1=val1-arg \ # overrides key1 from dev/values.yml + --data-value-yaml key2.nested=123 \ # merges into key2 from dev/values.yml + --data-value-yaml 'key3.other={"nested": true}' \ # decoded to a map + --data-value-file key4=client.crt \ # contains contents of client.crt + --data-values-env STR_VALS \ # decodes STR_VALS_key6 to a string + --data-values-env-yaml YAML_VALS \ # decodes YAML_VALS_key7 to a boolean + --data-values-file dev/ \ # finds and uses dev/values.yml + --data-values-inspect +``` + +yields: +```yaml +key2: + original: from values.yml + nested: 123 +key6: "true" +key7: true +key1: val1-arg +key3: + other: + nested: true +key4: +``` + +**Notes** +- the `--data-value...` flags can be repeated multiple times and used in any combination. The do not necessarily combine in the order supplied on the command-line; see [Data Values merge order](#data-values-merge-order) for details. +- Where schema is used, `--data-value...` flags can only _override_ values allowed by schema. Once schema is in use, _all_ Data Values must be declared in schema files. +- Where schema is _not_ used, (as of v0.34.0) Data Values can be _declared_ via `--data-value...` flags (previously, these flags could only _override_ previously declared values). This is useful for ad-hoc templating situations (usually a shell script one-liner) involving one or two Data Values — where type checks and validations are less useful. + + +#### `--data-values-file` + +Sets one or more Data Values from a plain YAML file, a directory containing YAML files, an HTTP URL, or standard input. + +``` +--data-values-file [@lib:]path +``` + +- `path` — one of: a filesystem path; an HTTP URL; or the dash character `-`. + - filesystem path can either be the path to a single YAML file _or_ a directory that (recursively) contains one or more files that have either the `.yaml` or `.yml` extension. If a directory is specified, all non-YAML files are ignored. + - HTTP URL is expected to resolve to a stream of one or more YAML document(s). + - `-` is the UNIX convention for referring to standard input. When specified, stdin is expected to contain a plain YAML document. + - The `-` file can only be used once in a `ytt` invocation. + - this input is given the name `stdin.yml`. +- `@lib:` — (optional) names the library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) rather than the root library. + +Not to be confused with [`--data-value-file`](#--data-value-file) which sets the value of exactly one Data Value. + +**Plain YAML Data Values** + +Regardless the source, the inputs for this flag must be plain YAML. It must not contain `ytt` templating (i.e. comments that start with `#@`. + +When multiple YAML documents are given, they are merged: +- each document is merged into the previous; +- map values are merged (values for entries that were previously defined are merged within the same key, recursively; values for new keys are added to the map); +- array values are replaced (last wins) + +**Examples:** + +_Example 1: Single File_ + +`prod-values.yml` +```yaml +domain: example.com +client_opts: + timeout: 10 + retry: 5 +``` + +```console +$ ytt ... --data-values-file prod-values.yml +``` + +sets all three Data Values: +- `domain=example.com` (a string value) +- `client_opts.timeout=10` (an integer value) +- `client_opts.retry=5` (an integer value) + +_Example 2: Directory_ + +See https://github.com/carvel-dev/ytt/tree/develop/examples/data-values-directory for a complete example and explanation. + +_Example 3: HTTP URL_ + +Given https://raw.githubusercontent.com/carvel-dev/ytt/develop/examples/data-values/values-file.yml + +```console +$ ytt --data-values-file https://raw.githubusercontent.com/carvel-dev/ytt/develop/examples/data-values/values-file.yml --data-values-inspect +``` +yields +```yaml +nothing: something +string: str +bool: true +int: 124 +new_thing: new +``` + +(Note the merged value of `int:`) + +#### `--data-value` + +Sets a single Data Value to a _string_ value. + +``` +--data-value [@lib:]key=value +``` + +- `key` — name of Data Value. Use dot notation for nested values (e.g. `key2.nested=val`) +- `value` — value to set (always interpreted as a string) +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- examples: `instance.count=123`, `key=string`, `input=true`, all set to strings + +#### `--data-value-yaml` + +Sets a single Data Value to a YAML-parsed value. + +``` +--data-value-yaml [@lib:]key=value +``` +- `key` — name of Data Value. +- `value` — value to set (decoded as a YAML value) +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- examples: `instance.count=123` sets as integer, `key=string` as string, `input=true` as bool + +#### `--data-value-file` + +Sets a single Data Value to the _contents_ of: a given file, an HTTP URL, or standard input. + +``` +--data-value-file [@lib:]key=path +``` +- `key` — name of Data Value. +- `path` — one of: a file path; an HTTP URL; or the dash character `-`. + - `-` is the UNIX convention for referring to standard input. The `-` file can only be used once in a `ytt` invocation. +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) + +This flag is particularly useful for loading multi-line string values from files such as private and public key files, certificates, etc. + +Not to be confused with [`--data-values-file`](#--data-values-file) which sets the values of multiple Data Values. + + +#### `--data-values-env` + +Sets one or more Data Values to _string_ values from OS environment variables that start with the given prefix. + +``` +--data-values-env [@lib:]PREFIX +``` +- `PREFIX` — the literal prefix used to select the set of environment variables from which to configure Data Values. +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) + +**Setting Environment Variables** +- for nested values, use double-underscore (i.e. `__`) in environment variable names to denote a "dot". + +**Examples** + +_Example: With Nested Values_ + +```shell +$ env +... +DVAL_key1=blue +DVAL_key2__nested=1337 +... +``` + +```console +$ ytt ... --data-values-env DVAL +``` + +would set two Data Values: +- `key1=blue` (the string "blue") +- `key2.nested="1337"` (the string "1337") + +#### `--data-values-env-yaml` + + +Sets one or more Data Values to the YAML-decoded values from OS environment variables that start with the given prefix. + +``` +--data-values-env-yaml [@lib:]PREFIX +``` + +- `PREFIX` — the literal prefix used to select the set of environment variables from which to configure Data Values. +- `@lib:` — (optional) specify library whose data values to configure (details [below](#setting-library-values-via-command-line-flags)) +- for nested values, use double-underscore (i.e. `__`) in environment variable names to denote a "dot". + +**Examples** + +_Example: With Nested Values_ + +```shell +$ env +... +DVAL_key1=blue +DVAL_key2__nested=1337 +... +``` + +```console +$ ytt ... --data-values-env DVAL +``` + +would set two Data Values: +- `key1=blue` (a string value) +- `key2.nested=1337` (an integer value) + + +### Configuring Data Values via Data Values Overlays + +Data Values can also be configured via a specific kind of `ytt` Overlay. + +A Data Values Overlay is a YAML document annotated with `@data/values`. + +```yaml +#@data/values +--- +key1: val1 +key2: + nested: val2 +key3: val3 +key4: +``` + +Note: +- `data.values` is a [`struct`](lang-ref-structs.md). + +#### Splitting Data Values Overlays into multiple files + +Available in v0.13.0. + +It's possible to split data values into multiple files (or specify multiple data values in the same file). `@ytt:data` library provides access to the _merged_ result. Merging is controlled via [overlay annotations](lang-ref-ytt-overlay.md) and follows same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order). Example: + +`values-default.yml`: + +```yaml +#@data/values +--- +key1: val1 +key2: + nested: val2 +key3: +key4: +``` + +`values-production.yml`: + +```yaml +#@data/values +--- +key3: new-val3 +#@overlay/remove +key4: +#@overlay/match missing_ok=True +key5: new-val5 +``` + +Note that `key4` is being removed, and `key5` is marked as `missing_ok=True` because it doesn't exist in `values-default.yml` (this is a safety feature to prevent accidental typos in keys). + +`config.yml`: + +```yaml +#@ load("@ytt:data", "data") + +first: #@ data.values.key1 +third: #@ data.values.key3 +fifth: #@ data.values.key5 +``` + +Running `ytt -f .` (or `ytt -f config.yml -f values-default.yml -f values-production.yml`) results in: + +```yaml +first: val1 +third: new-val3 +fifth: new-val5 +``` + +See [Multiple data values example](/ytt/#example:example-multiple-data-values) in the online playground. + + +### Data Values merge order + +Data values are merged in following order (latter one wins): + +1. default values from `@data/values-schema` files +2. `@data/values` overlays (same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order)) +3. `--data-values-file` (same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order)) +4. `--data-values-env` specified values (left to right) +5. `--data-values-env-yaml` specified values (left to right) +6. `--data-value` specified value (left to right) +7. `--data-value-yaml` specified value (left to right) +8. `--data-value-file` specified value (left to right) + +_(When configuring libraries, the [data values merge order](#library-data-values-merge-order), is the same, even if through different mechanisms.)_ + +--- +## Library data values + +Available in v0.28.0+ + +Each library may specify data values which will be evaluated separately from the root level library. + +### Setting library values via files + +To override library data values, add [`@library/ref`](lang-ref-ytt-library.md#libraryref) annotation to data values YAML document, like so: + +```yaml +#@library/ref "@lib1" +#@data/values +--- +key1: val1 +key2: val2 + +#@library/ref "@lib1" +#@data/values after_library_module=True +--- +key3: val3 +``` + +The `@data/values` annotation also supports a keyword argument `after_library_module`. If this keyword argument is specified, given data values will take precedence over data values passed to the `.with_data_values(...)` function when evaluating via the [library module](./lang-ref-ytt-library.md). + +### Setting library values via command line flags + +Data value flags support attaching values to libraries for use during [library module](./lang-ref-ytt-library.md) evaluation: + +```bash +export STR_VALS_key6=true # will be string 'true' +export YAML_VALS_key6=true # will be boolean true + +$ ytt -f . \ + --data-value @lib1:key1=val1-arg \ + --data-value-yaml @lib2:key2.nested=123 \ # will be int 123 + --data-value-yaml '@lib3:key3.other={"nested": true}' \ + --data-value-file @lib4:key4=/path \ + --data-values-env @lib5:STR_VALS \ + --data-values-env-yaml @lib6:YAML_VALS + --data-values-file @lib6:/path +``` + +```console +export STR_VALS_key6=true +export YAML_VALS_key7=true + +$ cat dev/values.yml +key1: values.yml-key1 +key2: + original: from values.yml + +$ ytt \ + --data-value @lib1:key1=val1-arg \ # overrides key1 from dev/values.yml + --data-value-yaml @lib1:key2.nested=123 \ # merges into key2 from dev/values.yml + --data-value-yaml '@lib2:key3.other={"nested": true}' \ # decoded to a map + --data-value-file @lib2:key4=client.crt \ # contains contents of client.crt + --data-values-env @lib2:STR_VALS \ # decodes STR_VALS_key6 to a string + --data-values-env-yaml @lib1:YAML_VALS \ # decodes YAML_VALS_key7 to a boolean + --data-values-file @lib1:dev/ # finds and uses dev/values.yml +``` +sends the following Data Values to library `lib1`: +```yaml +key2: + original: from values.yml + nested: 123 +key7: true +key1: val1-arg +``` +and the following Data Values to library `lib2`: +```yaml +key6: "true" +key3: + other: + nested: true +key4: +``` + + +### Library Data Values merge order + +For a given library instance, data values are merged in following order (latter one wins): + +1. default values from schema: + 1. `@data/values-schema` files within the library + 2. `@data/values-schema` files [externally referenced in](#setting-library-values-via-files). + 3. given through [`instance.with_data_values_schema()`](lang-ref-ytt-library.md#instancewith_data_values_schema) + 4. `@data/values-schema` files [externally referenced in `after_library_module=True`](#setting-library-values-via-files). +2. values from data value sources: + 1. `@data/values` overlays within the library (same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order)) + 2. `@data/values` overlays [externally referenced in](#setting-library-values-via-files) + 3. specified using [`instance.with_data_values()`](lang-ref-ytt-library.md#instancewith_data_values) + 4. `@data/values` overlays [externally referenced in `after_library_module=True`](#setting-library-values-via-files) + 5. `--data-values-file` specified files [referenced in](#setting-library-values-via-command-line-flags) (same ordering as [overlays](lang-ref-ytt-overlay.md#overlay-order)) + 6. `--data-values-env` specified values [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 7. `--data-values-env-yaml` specified values [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 8. `--data-value` specified value [referenced in](#setting-library-values-via-command-line-flags) (left to right) + 9. `--data-value-yaml` specified value [referenced in](#setting-library-values-via-command-line-flags) (left to right) + diff --git a/site/content/ytt/docs/v0.50.x/ytt-overlays.md b/site/content/ytt/docs/v0.50.x/ytt-overlays.md new file mode 100644 index 000000000..e3bec98f7 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/ytt-overlays.md @@ -0,0 +1,77 @@ +--- +aliases: [/ytt/docs/latest/ytt-overlays] +title: Overlays +--- + +## Overview + +Configure patch-like edits in `ytt` via "Overlays". + +_(For a high-level overview of `ytt`, see [How it works](how-it-works.md).)_ + +Sometimes it makes more sense to patch some YAML rather than template it. + +For example, when: +- the file should not be edited directly (e.g. from a third party); +- the edit will apply to most or all documents; or +- the specific variable is less commonly configured. + +## Example +Given a sample target YAML file: + +`config.yml` +```yaml +--- +id: 1 +contents: +- apple +--- +id: 2 +contents: +- pineapple +``` +... this overlay ... + +`add-content.yml` +```yaml +#@ load("@ytt:overlay", "overlay") + +#@overlay/match by=overlay.all, expects="1+" +--- +contents: +- pen +``` + +_read as..._ +1. _"match all YAML documents, expecting to match _at least_ one;"_ +2. _"within _each_ document, merge the key named `contents`;"_ +3. _"append an array item with the content `"pen"`"_ + + +... when processed by `ytt` ... + +```console +$ ytt -f config.yml -f add-content.yml +``` + +... produces ... + +`config.yml` _(edited)_ +```yaml +id: 1 +contents: +- apple +- pen +--- +id: 2 +contents: +- pineapple +- pen +``` + +## Next Steps + +- [Overlay example](/ytt/#example:example-overlay-files) in the ytt Playground to try it out, yourself. +- [ytt Library: Overlay module](lang-ref-ytt-overlay.md) for reference of all overlay annotations and functions. +- [Data Values vs Overlays](data-values-vs-overlays.md) for when to use one mechanism over the other. +- [Primer on `ytt` Overlays](/blog/primer-on-ytt-overlays/) for a screencast-formatted in-depth introduction to writing and using overlays. diff --git a/site/content/ytt/docs/v0.50.x/ytt-playground-screenshot.png b/site/content/ytt/docs/v0.50.x/ytt-playground-screenshot.png new file mode 100644 index 000000000..92882a6a9 Binary files /dev/null and b/site/content/ytt/docs/v0.50.x/ytt-playground-screenshot.png differ diff --git a/site/content/ytt/docs/v0.50.x/ytt-text-templating.md b/site/content/ytt/docs/v0.50.x/ytt-text-templating.md new file mode 100644 index 000000000..f2e588508 --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/ytt-text-templating.md @@ -0,0 +1,54 @@ +--- +aliases: [/ytt/docs/latest/ytt-text-templating] +title: Text Templating +--- + +## Overview + +ytt supports text templating within YAML strings and `.txt` files. + +Text templating is controlled via `(@` and `@)` directives. These directives can be combined with following markers: + +- `=` to output result; result must be of type string +- `-` to trim space either to the left (if next to opening directive) or right (if next to closing directive) + +Examples: + +- `before (@ 123 @) middle (@= "tpl" @) after` produces `before middle tpl after` +- `before (@- 123 -@) middle (@-= "tpl" -@) after` produces `beforemiddletplafter` + +## Inside YAML strings + +`+` operand or [`string·format(...)`](lang-ref-string.md) method provide a good way to build strings: + +- `name: #@ name_prefix + "-secret"` +- `name: #@ name_prefix + "-" + str(1234)` +- `name: #@ "{}-secret-{}".format(name_prefix, name_suffix)` + +However, occasionally it might be useful to use text templating directly in YAML strings. To do so YAML node must be annotated with `@yaml/text-templated-strings` (v0.17.0+). Annotation will apply to node and its child nodes. + +Examples: + +- basic use +```yaml +#@ val1 = "val1" +#@ val2 = "val2" + +#@yaml/text-templated-strings +templated: "before (@= val1 @) middle (@= val2 @) after" + +non_templated: "(@ something" +``` + +- nested nodes +```yaml +#@ val1 = "val1" +#@ val2 = "val2" + +#@yaml/text-templated-strings +--- +key: + nested_key_(@= val1 @): "middle (@= val2 @) after" +``` + +See [Text template example](/ytt/#example:example-text-template) in online playground. diff --git a/site/content/ytt/docs/v0.50.x/ytt-vs-x.md b/site/content/ytt/docs/v0.50.x/ytt-vs-x.md new file mode 100644 index 000000000..fbafe2fcd --- /dev/null +++ b/site/content/ytt/docs/v0.50.x/ytt-vs-x.md @@ -0,0 +1,61 @@ +--- +aliases: [/ytt/docs/latest/ytt-vs-x] +title: ytt vs x +--- + + +## ytt vs Go text/template (and other text templating tools) + +- [Go's text/template](https://golang.org/pkg/text/template/) +- [Jinja](http://jinja.pocoo.org/) + +Most generic templating tools do not understand content that they are templating and consider it just plain text. ytt operates on YAML structures, hence typical escaping and formatting problems common to text templating tools are eliminated. Additionally, ytt provides a very easy way to make structures reusable in a much more readable way that's possible with some text templating tools. + +## ytt vs jsonnet + +- [Jsonnet](https://jsonnet.org/) + +ytt conceptually is very close to [jsonnet](https://jsonnet.org/). Both operate on data structures instead of text, hence are able to provide a better way to construct, compose and reuse structures. jsonnet introduces a custom language to help perform structure operations. ytt on the other hand, builds upon a Python-like language, which we think will be more familiar to the larger community. + +We also believe that transitioning from plain YAML to templated YAML with `ytt` is very easy and natural. + +## ytt vs Dhall + +- [Dhall](https://dhall-lang.org/) + +Dhall language is a configuration language that can output YAML, and JSON. One of its strong points is ability to provide scripting environment that is "hermetically sealed" and safe, even against malicious templates. `ytt` also embraces same goal (and builds upon the great work of Starlark community) by exposing small API in the template context. For example, there is no way to make network calls, read from file system, _or currently, even get time_. + +## ytt vs Kustomize (and CF BOSH ops files) + +- [Kustomize](https://kubernetes.io/blog/2018/05/29/introducing-kustomize-template-free-configuration-customization-for-kubernetes/) +- [CF BOSH's ops files](https://bosh.io/docs/cli-ops-files) + +Configuration customization tools are unique in a sense that they don't allow templating but rather build upon "base" configuration. `ytt` offers its own take on configuration customization via the ['overlay' feature](lang-ref-ytt-overlay.md). Unlike other tools, overlay operations (remove, replace, merge) in `ytt` mimic structure of the base configuration. For example in Kustomize to remove a particular map key, one has to use JSON patch syntax which is quite different from the normal document structure. On the other hand, `ytt` uses its ability to annotate YAML structures, hence it can mark map key that should be deleted. All in all, we think that `ytt`'s approach is superior. + +Here are a few more detailed differences: + +- `ytt` overlays + + - are not Kubernetes-specific so various types of configurations are covered that kustomize cannot deal with. + - cover all CRUD operations in one consistent style whereas kustomize needs varied syntaxes for varied types of modification (SMP vs jsonPatch, etc.). + - do not care about native kinds, CRDs vs something else since they are generic. + +- `ytt` is not just an overlay tool; it supports overlaying _and_ templating. We see configuration writing split into two categories: configuration authors, and configuration consumers. Configuration _authors_ are best supported by templating; however, configuration _consumers_ typically need more than just templating inputs. Overlaying provides the rest. Having one tool with consistent functionality across templating and overlays is powerful. +- `ytt` is more explicit about missing map keys, etc (avoids a lot of unnecessary typos early on). +- `ytt` allows you to define variables. +- `ytt` has facilties to inject data into overlays from a variety of inputs including command line arguments, environment variables, and files. + +## ytt vs Orchestration Tools (Pulumi / HELM) + +- [Pulumi](https://www.pulumi.com/) +- [HELM](https://helm.sh/) + +Orchestration tools like Pulumi, and HELM, have combined configuration management and workflow management into the same tool. There are advantages and disadvantages to that. `ytt` is designed specifically to only focus on configuration management. Though, YAML output can be used with HELM, Pulumi, or other tools. + +## ytt vs plain Ruby/Python/etc + +Key advantages for `ytt`: + +- provides an easy way to operate on structures (maps, lists, etc.). One can definitely use a regular language to do data manipulation. However, this is not what the language is optimized for, especially if data is heavily nested, typically leading to very verbose and less readable code. +- provides an _easy and safe_ way to execute templates without worrying that template code may be malicious. One can Dockerize execution of regular language templates but of course that brings in pretty heavy dependency. +- provides an _easy_ way to customize any part of configuration via [overlays](lang-ref-ytt-overlay.md). This is not possible to do with a regular language without parameterizing everything (a general anti-pattern) or bringing in an additional tool (e.g. BOSH ops files). diff --git a/site/data/ytt/docs/toc-mapping.yml b/site/data/ytt/docs/toc-mapping.yml index a29fc1081..3175f3995 100644 --- a/site/data/ytt/docs/toc-mapping.yml +++ b/site/data/ytt/docs/toc-mapping.yml @@ -11,3 +11,4 @@ v0.46.x: ytt-v0-46-x-toc v0.47.x: ytt-v0-47-x-toc v0.48.0: ytt-v0-48-0-toc v0.49.x: ytt-v0-49-x-toc +v0.50.x: ytt-v0-50-x-toc diff --git a/site/data/ytt/docs/ytt-v0-50-x-toc.yml b/site/data/ytt/docs/ytt-v0-50-x-toc.yml new file mode 100644 index 000000000..24feeb540 --- /dev/null +++ b/site/data/ytt/docs/ytt-v0-50-x-toc.yml @@ -0,0 +1,115 @@ +toc: + - title: Introduction + subfolderitems: + - page: About ytt + url: / + - page: Install + url: /install + - page: How it Works + url: /how-it-works + - title: Concepts + subfolderitems: + - page: YAML and Annotations + url: /yaml-primer + - title: Quick References + subfolderitems: + - page: Schema Validations Cheat Sheet + url: /schema-validations-cheat-sheet + - title: How To + subfolderitems: + - page: Get Started & Modularize code + url: /how-to-modularize + - page: Use Data Values + url: /how-to-use-data-values + - page: Write Schema + url: /how-to-write-schema + - page: Write Schema Validations + url: /how-to-write-validations + - page: Export Schema in OpenAPI format + url: /how-to-export-schema + - page: Use Overlays + url: /ytt-overlays + - page: Data Values vs. Overlays + url: /data-values-vs-overlays + - page: Inject secrets + url: /injecting-secrets + - title: Reference + subfolderitems: + - page: Language + url: /lang + - page: Annotations + url: /lang-ref-annotation + - page: Data Values + url: /ytt-data-values + - page: Data Values Schema + url: /lang-ref-ytt-schema + - page: Text Templating + url: /ytt-text-templating + - title: Built-in @ytt library + subfolderitems: + - page: All modules + url: /lang-ref-ytt + - page: Assert module + url: /lang-ref-ytt-assert + - page: Library module + url: /lang-ref-ytt-library + - page: Overlay module + url: /lang-ref-ytt-overlay + - page: Struct module + url: /lang-ref-ytt-struct + - page: Template module + url: /lang-ref-ytt-template + - page: Version module + url: /lang-ref-ytt-version + - title: Language + subfolderitems: + - page: General + url: /lang + - page: String + url: /lang-ref-string + - page: List + url: /lang-ref-list + - page: Dictionary + url: /lang-ref-dict + - page: Struct + url: /lang-ref-structs + - page: YAML Fragment + url: /lang-ref-yaml-fragment + - page: If conditional + url: /lang-ref-if + - page: For loop + url: /lang-ref-for + - page: Function + url: /lang-ref-def + - page: Load + url: /lang-ref-load + - title: CLI configuration + subfolderitems: + - page: Inputs + url: /inputs + - page: Outputs + url: /outputs + - page: File Marks + url: /file-marks + - page: Strict yaml + url: /strict + - title: FAQ + subfolderitems: + - page: FAQ + url: /faq + - page: Why ytt vs. x? + url: /ytt-vs-x + - page: Schema Migration Guide + url: /data-values-schema-migration-guide + - page: Code of Conduct + shared_url: /code-of-conduct + - page: Contributing + shared_url: /contributing + - page: Carvel Development Guidelines + shared_url: /development_guidelines + - page: Security + shared_url: /security-policy + - page: Known Limitations + url: /known-limitations + - page: Playground Examples + url: /index-playground-examples