diff --git a/charts/_source/templates/prometheusConfiguration.yaml b/charts/_source/templates/prometheusConfiguration.yaml new file mode 100644 index 0000000..c32efdb --- /dev/null +++ b/charts/_source/templates/prometheusConfiguration.yaml @@ -0,0 +1,14 @@ +{{- if eq .Values.format "PrometheusConfigurationFile" }} +groups: + - name: {{ $.Values.rulesGroupName }} + rules: +{{- range $name, $val := .Values.rules }} +{{- $_ := set . "name" $name }} +{{- $_ := set . "globalAdditionalExprLabels" $.Values.global.additionalExprLabels }} +{{- $_ := set . "globalAdditionalRuleLabels" $.Values.global.additionalRuleLabels }} +{{- $_ := set . "defaultRunbookUrl" $.Values.defaultRunbookUrl }} +{{- $_ := set . "chartVersion" $.Chart.Version }} + +{{- include "chart.renderPrometheusRule" . | indent 8 -}} +{{- end }} +{{- end }} diff --git a/charts/_source/templates/prometheusRules.yaml b/charts/_source/templates/prometheusRules.yaml index 1138105..891e8c3 100644 --- a/charts/_source/templates/prometheusRules.yaml +++ b/charts/_source/templates/prometheusRules.yaml @@ -1,3 +1,4 @@ +{{- if eq .Values.format "PrometheusRuleCRD" }} {{- $fullName := include "chart.fullname" . -}} --- apiVersion: monitoring.coreos.com/v1 @@ -22,3 +23,4 @@ spec: {{- include "chart.renderPrometheusRule" . | indent 8 -}} {{- end }} +{{- end }} diff --git a/charts/_source/tests/prometheusRules_test.yaml b/charts/_source/tests/prometheusRules_test.yaml index 4722eca..f9873ae 100644 --- a/charts/_source/tests/prometheusRules_test.yaml +++ b/charts/_source/tests/prometheusRules_test.yaml @@ -1,9 +1,9 @@ # yaml-language-server: $schema=https://raw.githubusercontent.com/helm-unittest/helm-unittest/main/schema/helm-testsuite.json suite: prometheus rules tests -templates: - - prometheusRules.yaml tests: - it: render default deployment + templates: + - prometheusRules.yaml values: - ./values/with_rules.yaml asserts: @@ -34,7 +34,41 @@ tests: - lengthEqual: path: spec.groups[0].rules count: 2 + - it: render to Prometheus configuration file format + templates: + - prometheusConfiguration.yaml + values: + - ./values/with_rules.yaml + - ./values/with_PrometheusConfigurationFileFormat.yaml + asserts: + - equal: + path: groups[0].name + value: unittest + - equal: + path: groups[0].rules[0].alert + value: Rule1 + - equal: + path: groups[0].rules[0].expr + value: | + metric1{} > 42 + - equal: + path: groups[0].rules[0].for + value: 10m + - equal: + path: groups[0].rules[0].labels.priority + value: P1 + - equal: + path: groups[0].rules[0].annotations.summary + value: "Metric 1 is over 42" + - equal: + path: groups[0].rules[0].annotations.runbook_url + value: https://acme.com/runbooks/0.0.0/Rule1.md + - lengthEqual: + path: groups[0].rules + count: 2 - it: render with global additional labels on expression + templates: + - prometheusRules.yaml values: - ./values/with_rules.yaml - ./values/with_globalAdditionalExprLabels.yaml @@ -44,6 +78,8 @@ tests: value: | metric1{environment=production,} > 42 - it: render with global additional labels on rules + templates: + - prometheusRules.yaml values: - ./values/with_rules.yaml - ./values/with_globalAdditionalRuleLabels.yaml @@ -52,6 +88,8 @@ tests: path: spec.groups[0].rules[0].labels.team value: sre - it: render with global additional labels on rules + templates: + - prometheusRules.yaml values: - ./values/with_rules.yaml - ./values/with_low_priority.yaml @@ -60,6 +98,8 @@ tests: path: spec.groups[0].rules[0].labels.priority value: P3 - it: render with additional additionalPrometheusRuleLabels + templates: + - prometheusRules.yaml values: - ./values/with_additionalPrometheusRuleLabels.yaml asserts: diff --git a/charts/_source/tests/values/with_PrometheusConfigurationFileFormat.yaml b/charts/_source/tests/values/with_PrometheusConfigurationFileFormat.yaml new file mode 100644 index 0000000..2e52b6f --- /dev/null +++ b/charts/_source/tests/values/with_PrometheusConfigurationFileFormat.yaml @@ -0,0 +1 @@ +format: PrometheusConfigurationFile diff --git a/charts/_source/values.yaml b/charts/_source/values.yaml index d5f52fb..13e161c 100644 --- a/charts/_source/values.yaml +++ b/charts/_source/values.yaml @@ -2,6 +2,8 @@ global: additionalExprLabels: [] additionalRuleLabels: {} +format: PrometheusRuleCRD + rulesGroupName: default rules: {} diff --git a/charts/prometheus-postgresql-alerts/values.yaml b/charts/prometheus-postgresql-alerts/values.yaml index 85099b5..931febf 100644 --- a/charts/prometheus-postgresql-alerts/values.yaml +++ b/charts/prometheus-postgresql-alerts/values.yaml @@ -1,6 +1,8 @@ global: additionalRuleLabels: {} +format: PrometheusRuleCRD + defaultRunbookUrl: https://qonto.github.io/database-monitoring-framework/{{chartVersion}}/runbooks/postgresql/{{alertName}} rulesGroupName: postgresql.rules diff --git a/charts/prometheus-rds-alerts/values.yaml b/charts/prometheus-rds-alerts/values.yaml index 3e6776d..b2f1fd8 100644 --- a/charts/prometheus-rds-alerts/values.yaml +++ b/charts/prometheus-rds-alerts/values.yaml @@ -1,6 +1,8 @@ global: additionalRuleLabels: {} +format: PrometheusRuleCRD + defaultRunbookUrl: https://qonto.github.io/database-monitoring-framework/{{chartVersion}}/runbooks/rds/{{alertName}} rulesGroupName: rds.rules diff --git a/content/faq.md b/content/faq.md index 7c0247c..ab25819 100644 --- a/content/faq.md +++ b/content/faq.md @@ -15,6 +15,14 @@ This project was initiated by [Qonto's](https://qonto.com/) SRE engineers to sha Yes, we took lots of inspiration on [Prometheus-operator runbook](https://github.com/prometheus-operator/runbooks), which provides similar experience for Prometheus & Kubernetes technologies. +## Deployment + +### Can I deploy the database monitoring framework on Prometheus standalone deployment? + +Project was designed for Kubernetes deployment using Prometheus operator, but you can use for standalone deployment. + +Use the `format=PrometheusConfigurationFile` Helm parameter to render rules as Prometheus rule files. + ## Alerts ### Customize alerts diff --git a/content/getting-started.md b/content/getting-started.md index c057e26..f7673a4 100644 --- a/content/getting-started.md +++ b/content/getting-started.md @@ -19,26 +19,86 @@ See also our tutorials for concrete deployment example. Steps to deploy the framework on on your infrastructure: -1. Deploy Prometheus operator +{{< tabs "methods" >}} +{{< tab "Prometheus Kubernetes Operator" >}} +Requirements - Make sure: - - `serviceMonitorSelector` is defined to enable `ServiceMonitor` ([documentation](https://github.com/prometheus-operator/prometheus-operator/blob/v0.68.0/Documentation/user-guides/alerting.md#deploying-prometheus-rules)) - - `ruleSelector` is enabled for `PrometheusRules` ([documentation](https://github.com/prometheus-operator/prometheus-operator/blob/v0.68.0/Documentation/user-guides/alerting.md#deploying-prometheus-rules)) +- [Helm](https://helm.sh/docs/intro/install/) -1. Deploy Prometheus RDS exporters and/or PostgreSQL exporter as describe in tutorials +- [Prometheus Kubernetes operator](https://github.com/prometheus-operator/prometheus-operator) + + 1. `ruleSelector` must be enabled ([documentation](https://github.com/prometheus-operator/prometheus-operator/blob/v0.68.0/Documentation/user-guides/alerting.md#deploying-prometheus-rules)) + + 1. `serviceMonitorSelector` must be enabled for `ServiceMonitor` ([documentation](https://github.com/prometheus-operator/prometheus-operator/blob/v0.68.0/Documentation/user-guides/alerting.md#deploying-prometheus-rules)) + +Steps + +1. Deploy [Prometheus RDS exporter]({{< ref "/tutorials/rds/exporter-deployment" >}}) and/or [Prometheus PostgreSQL exporter]({{< ref "/tutorials/postgresql/exporter-deployment" >}}) 1. Deploy charts with helm ```bash # RDS alerts - helm install prometheus-rds-alerts-chart oci://public.ecr.aws/qonto/prometheus-rds-alerts-chart:{{% current_version %}} --namespace ${KUBERNETES_NAMESPACE} + helm install prometheus-rds-alerts-chart oci://public.ecr.aws/qonto/prometheus-rds-alerts-chart \ + --version {{% current_version %}} \ + --namespace monitoring ``` ```bash # PostgreSQL alerts - helm install prometheus-postgresql-alerts-chart oci://public.ecr.aws/qonto/prometheus-postgresql-alerts-chart:{{% current_version %}} --namespace ${KUBERNETES_NAMESPACE} + helm install prometheus-postgresql-alerts-chart oci://public.ecr.aws/qonto/prometheus-postgresql-alerts-chart \ + --version {{% current_version %}} \ + --namespace monitoring + ``` + +1. Connect to Prometheus to check that the rules are correctly loaded (`https:///rules`) + +{{< /tab >}} +{{< tab "Prometheus standalone server" >}} + +Requirements + +- [Helm](https://helm.sh/docs/intro/install/) + +- [Prometheus server](https://prometheus.io/docs/prometheus/latest/installation/) + +Steps + +1. Deploy [Prometheus RDS exporter]({{< ref "/tutorials/rds/exporter-deployment" >}}) and/or [Prometheus PostgreSQL exporter]({{< ref "/tutorials/postgresql/exporter-deployment" >}}) + +1. Generate Prometheus rule configuration files + + ```bash + # Generate RDS rules in /etc/prometheus/rds.rules.yaml + helm template oci://public.ecr.aws/qonto/prometheus-rds-alerts-chart \ + --version {{% current_version %}} \ + --set format=PrometheusConfigurationFile \ + > /etc/prometheus/rds.rules.yaml + ``` + + ```bash + # Generate PostgreSQL rules in /etc/prometheus/postgresql.rules.yaml + helm template oci://public.ecr.aws/qonto/prometheus-postgresql-alerts-chart \ + --version {{% current_version %}} \ + --set format=PrometheusConfigurationFile \ + > /etc/postgresql.rules.yaml ``` +1. Add files to the `rule_files` parameter in the Prometheus configuration file + + ```yaml + rule_files: + - /etc/prometheus/rds.rules.yaml + - /etc/prometheus/postgresql.rules.yaml + ``` + +1. Reload Prometheus configuration + +1. Connect to Prometheus to check that the rules are correctly loaded (`https:///rules`) + +{{< /tab >}} +{{< /tabs >}} + ## Charts Prometheus alerts are available as Helm chart: