From 72bfea085f4bf41e7813d2ed405fd4e7fdb4ee81 Mon Sep 17 00:00:00 2001 From: Damien Ciabrini Date: Fri, 11 Aug 2023 10:28:20 +0000 Subject: [PATCH] galera: support mariabackup SST method Make the SST method configurable in the galera custom resource, and allow both rsync and mariabackup methods. Mariabackup requires a database user's credentials to operate, so reuse the root db user for the time being. The ability to inject passwords in template galera config files has been implemented. We benefit from that to generate the usual .my.cnf config file for mysql CLI running in the galera container. --- api/bases/mariadb.openstack.org_galeras.yaml | 7 + api/v1beta1/galera_types.go | 13 ++ .../bases/mariadb.openstack.org_galeras.yaml | 7 + .../samples/mariadb_v1beta1_galera_sst.yaml | 10 + pkg/mariadb/statefulset.go | 141 +------------- pkg/mariadb/volumes.go | 181 ++++++++++++++++++ templates/galera/bin/mysql_bootstrap.sh | 23 ++- templates/galera/bin/mysql_probe.sh | 2 +- templates/galera/config/config.json | 13 ++ .../config/galera_sst_mariabackup.cnf.in | 3 + templates/galera/config/init_config.json | 10 + templates/galera/config/my.cnf.in | 4 + 12 files changed, 270 insertions(+), 144 deletions(-) create mode 100644 config/samples/mariadb_v1beta1_galera_sst.yaml create mode 100644 templates/galera/config/galera_sst_mariabackup.cnf.in create mode 100644 templates/galera/config/init_config.json create mode 100644 templates/galera/config/my.cnf.in diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index 564c69d7..749a3adc 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -78,6 +78,13 @@ spec: secret: description: Name of the secret to look for password keys type: string + sst: + default: rsync + description: Snapshot State Transfer method to use for full node synchronization + enum: + - rsync + - mariabackup + type: string storageClass: description: Storage class to host the mariadb databases type: string diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index da3abaec..01cc1ac7 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -26,6 +26,7 @@ const ( CustomServiceConfigFile = "galera_custom.cnf.in" ) + // GaleraSpec defines the desired state of Galera type GaleraSpec struct { // Name of the secret to look for password keys @@ -56,8 +57,20 @@ type GaleraSpec struct { // +kubebuilder:validation:Optional // Adoption configuration AdoptionRedirect AdoptionRedirectSpec `json:"adoptionRedirect"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=rsync + // +kubebuilder:validation:Enum=rsync;mariabackup + // Snapshot State Transfer method to use for full node synchronization + SST GaleraSST `json:"sst"` } +// Supported SST type +type GaleraSST string +const ( + RSync GaleraSST = "rsync" + MariaBackup = "mariabackup" +) + // GaleraAttributes holds startup information for a Galera host type GaleraAttributes struct { // Last recorded replication sequence number in the DB diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index 564c69d7..749a3adc 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -78,6 +78,13 @@ spec: secret: description: Name of the secret to look for password keys type: string + sst: + default: rsync + description: Snapshot State Transfer method to use for full node synchronization + enum: + - rsync + - mariabackup + type: string storageClass: description: Storage class to host the mariadb databases type: string diff --git a/config/samples/mariadb_v1beta1_galera_sst.yaml b/config/samples/mariadb_v1beta1_galera_sst.yaml new file mode 100644 index 00000000..8b0d5c2d --- /dev/null +++ b/config/samples/mariadb_v1beta1_galera_sst.yaml @@ -0,0 +1,10 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +spec: + secret: osp-secret + storageClass: local-storage + storageRequest: 500M + replicas: 3 + sst: mariabackup diff --git a/pkg/mariadb/statefulset.go b/pkg/mariadb/statefulset.go index 2abcfff8..b7c4dc7d 100644 --- a/pkg/mariadb/statefulset.go +++ b/pkg/mariadb/statefulset.go @@ -53,25 +53,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraInitVolumeMounts(g), }}, Containers: []corev1.Container{{ Image: g.Spec.ContainerImage, @@ -84,16 +66,6 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, { Name: "KOLLA_CONFIG_STRATEGY", Value: "COPY_ALWAYS", - }, { - Name: "DB_ROOT_PASSWORD", - ValueFrom: &corev1.EnvVarSource{ - SecretKeyRef: &corev1.SecretKeySelector{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Spec.Secret, - }, - Key: "DbRootPassword", - }, - }, }}, Ports: []corev1.ContainerPort{{ ContainerPort: 3306, @@ -102,29 +74,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { ContainerPort: 4567, Name: "galera", }}, - VolumeMounts: []corev1.VolumeMount{{ - MountPath: "/var/lib/mysql", - Name: "mysql-db", - }, { - MountPath: "/var/lib/config-data", - ReadOnly: true, - Name: "config-data", - }, { - MountPath: "/var/lib/pod-config-data", - Name: "pod-config-data", - }, { - MountPath: "/var/lib/secrets", - ReadOnly: true, - Name: "secrets", - }, { - MountPath: "/var/lib/operator-scripts", - ReadOnly: true, - Name: "operator-scripts", - }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }}, + VolumeMounts: getGaleraVolumeMounts(g), StartupProbe: &corev1.Probe{ ProbeHandler: corev1.ProbeHandler{ Exec: &corev1.ExecAction{ @@ -149,92 +99,7 @@ func StatefulSet(g *mariadbv1.Galera) *appsv1.StatefulSet { }, }, }}, - Volumes: []corev1.Volume{ - { - Name: "secrets", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: g.Spec.Secret, - Items: []corev1.KeyToPath{ - { - Key: "DbRootPassword", - Path: "dbpassword", - }, - }, - }, - }, - }, - { - Name: "kolla-config", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, - }, - }, - { - Name: "pod-config-data", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "config-data", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "galera.cnf.in", - Path: "galera.cnf.in", - }, - { - Key: mariadbv1.CustomServiceConfigFile, - Path: mariadbv1.CustomServiceConfigFile, - }, - }, - }, - }, - }, - { - Name: "operator-scripts", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-scripts", - }, - Items: []corev1.KeyToPath{ - { - Key: "mysql_bootstrap.sh", - Path: "mysql_bootstrap.sh", - }, - { - Key: "mysql_probe.sh", - Path: "mysql_probe.sh", - }, - { - Key: "detect_last_commit.sh", - Path: "detect_last_commit.sh", - }, - { - Key: "detect_gcomm_and_start.sh", - Path: "detect_gcomm_and_start.sh", - }, - }, - }, - }, - }, - }, + Volumes: getGaleraVolumes(g), }, }, VolumeClaimTemplates: []corev1.PersistentVolumeClaim{ diff --git a/pkg/mariadb/volumes.go b/pkg/mariadb/volumes.go index 459131f3..a8a5228a 100644 --- a/pkg/mariadb/volumes.go +++ b/pkg/mariadb/volumes.go @@ -1,6 +1,7 @@ package mariadb import ( + mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" ) @@ -113,3 +114,183 @@ func getInitVolumeMounts() []corev1.VolumeMount { } } + +func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { + configTemplates := []corev1.KeyToPath{ + { + Key: "my.cnf.in", + Path: ".my.cnf.in", + }, + { + Key: "galera.cnf.in", + Path: "galera.cnf.in", + }, + { + Key: mariadbv1.CustomServiceConfigFile, + Path: mariadbv1.CustomServiceConfigFile, + }, + } + + if g.Spec.SST == mariadbv1.MariaBackup { + configTemplates = append(configTemplates, corev1.KeyToPath{ + Key: "galera_sst_mariabackup.cnf.in", + Path: "galera_sst_mariabackup.cnf.in", + }) + } + + volumes := []corev1.Volume{ + { + Name: "secrets", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: g.Spec.Secret, + Items: []corev1.KeyToPath{ + { + Key: "DbRootPassword", + Path: "dbpassword", + }, + }, + }, + }, + }, + { + Name: "kolla-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: []corev1.KeyToPath{ + { + Key: "config.json", + Path: "config.json", + }, + }, + }, + }, + }, + { + Name: "kolla-config-init", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: []corev1.KeyToPath{ + { + Key: "init_config.json", + Path: "config.json", + }, + }, + }, + }, + }, + { + Name: "pod-config-data", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "config-data", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-config-data", + }, + Items: configTemplates, + }, + }, + }, + { + Name: "operator-scripts", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: g.Name + "-scripts", + }, + Items: []corev1.KeyToPath{ + { + Key: "mysql_bootstrap.sh", + Path: "mysql_bootstrap.sh", + }, + { + Key: "mysql_probe.sh", + Path: "mysql_probe.sh", + }, + { + Key: "detect_last_commit.sh", + Path: "detect_last_commit.sh", + }, + { + Key: "detect_gcomm_and_start.sh", + Path: "detect_gcomm_and_start.sh", + }, + }, + }, + }, + }, + } + + return volumes +} + +func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config", + }, + } + + return volumeMounts +} + +func getGaleraInitVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { + volumeMounts := []corev1.VolumeMount{ + { + MountPath: "/var/lib/mysql", + Name: "mysql-db", + }, { + MountPath: "/var/lib/config-data", + ReadOnly: true, + Name: "config-data", + }, { + MountPath: "/var/lib/pod-config-data", + Name: "pod-config-data", + }, { + MountPath: "/var/lib/secrets", + ReadOnly: true, + Name: "secrets", + }, { + MountPath: "/var/lib/operator-scripts", + ReadOnly: true, + Name: "operator-scripts", + }, { + MountPath: "/var/lib/kolla/config_files", + ReadOnly: true, + Name: "kolla-config-init", + }, + } + + return volumeMounts +} diff --git a/templates/galera/bin/mysql_bootstrap.sh b/templates/galera/bin/mysql_bootstrap.sh index 522ee1fb..42bbc7d4 100755 --- a/templates/galera/bin/mysql_bootstrap.sh +++ b/templates/galera/bin/mysql_bootstrap.sh @@ -1,5 +1,10 @@ #!/bin/bash -set +eux +set -eu + +init_error() { + echo "Container initialization failed at $(caller)." >&2 +} +trap init_error ERR if [ -e /var/lib/mysql/mysql ]; then echo -e "Database already exists. Reuse it." @@ -18,12 +23,20 @@ fi # Generate the mariadb configs from the templates, these will get # copied by `kolla_start` when the pod's main container will start -PODNAME=$(hostname -f | cut -d. -f1,2) -PODIP=$(grep "${PODNAME}" /etc/hosts | cut -d$'\t' -f1) +export PODNAME=$(hostname -f | cut -d. -f1-4) +export PODIP=$(grep "${PODNAME}" /etc/hosts | cut -d$'\t' -f1) +read -s -u 3 3< <(cat /var/lib/secrets/dbpassword; echo) DB_ROOT_PASSWORD +read -s -u 3 3< <(cat /var/lib/secrets/dbpassword; echo) MARIABACKUP_PASSWORD +export DB_ROOT_PASSWORD MARIABACKUP_PASSWORD + cd /var/lib/config-data -for cfg in *.cnf.in; do +for cfg in *.cnf.in .*.cnf.in; do if [ -s "${cfg}" ]; then echo "Generating config file from template ${cfg}" - sed -e "s/{ PODNAME }/${PODNAME}/" -e "s/{ PODIP }/${PODIP}/" "/var/lib/config-data/${cfg}" > "/var/lib/pod-config-data/${cfg%.in}" + awk '{ +patsplit($0,vars,/{ (PODNAME|PODIP|DB_ROOT_PASSWORD|MARIABACKUP_PASSWORD) }/); +for(v in vars){ k=vars[v]; gsub(/\W/,"",k); gsub(vars[v], ENVIRON[k])}; +print $0 +}' "/var/lib/config-data/${cfg}" > "/var/lib/pod-config-data/${cfg%.in}" fi done diff --git a/templates/galera/bin/mysql_probe.sh b/templates/galera/bin/mysql_probe.sh index 4b190b1a..8ccf2c2c 100755 --- a/templates/galera/bin/mysql_probe.sh +++ b/templates/galera/bin/mysql_probe.sh @@ -2,7 +2,7 @@ set -eu # This secret is mounted by k8s and always up to date -read -s -u 3 3< /var/lib/secrets/dbpassword MYSQL_PWD || true +read -s -u 3 3< <(cat /var/lib/secrets/dbpassword; echo) MYSQL_PWD export MYSQL_PWD PROBE_USER=root diff --git a/templates/galera/config/config.json b/templates/galera/config/config.json index 185e33fa..e01486f6 100644 --- a/templates/galera/config/config.json +++ b/templates/galera/config/config.json @@ -1,6 +1,12 @@ { "command": "/usr/local/bin/detect_gcomm_and_start.sh", "config_files": [ + { + "source": "/var/lib/pod-config-data/.my.cnf", + "dest": "/etc/my.cnf.d/.my.cnf", + "owner": "mysql", + "perm": "0600" + }, { "source": "/var/lib/pod-config-data/galera.cnf", "dest": "/etc/my.cnf.d/galera.cnf", @@ -14,6 +20,13 @@ "perm": "0644", "optional": true }, + { + "source": "/var/lib/pod-config-data/galera_sst_mariabackup.cnf", + "dest": "/etc/my.cnf.d/galera_mariabackup.cnf", + "owner": "root", + "perm": "0644", + "optional": true + }, { "source": "/var/lib/operator-scripts", "dest": "/usr/local/bin", diff --git a/templates/galera/config/galera_sst_mariabackup.cnf.in b/templates/galera/config/galera_sst_mariabackup.cnf.in new file mode 100644 index 00000000..8420e61a --- /dev/null +++ b/templates/galera/config/galera_sst_mariabackup.cnf.in @@ -0,0 +1,3 @@ +[mysqld] +wsrep_sst_method = mariabackup +wsrep_sst_auth = root:{ MARIABACKUP_PASSWORD } diff --git a/templates/galera/config/init_config.json b/templates/galera/config/init_config.json new file mode 100644 index 00000000..7984f1fe --- /dev/null +++ b/templates/galera/config/init_config.json @@ -0,0 +1,10 @@ +{ + "command": "/usr/bin/true", + "permissions": [ + { + "path": "/var/lib/mysql", + "owner": "mysql:mysql", + "recurse": "true" + } + ] +} diff --git a/templates/galera/config/my.cnf.in b/templates/galera/config/my.cnf.in new file mode 100644 index 00000000..065d0f59 --- /dev/null +++ b/templates/galera/config/my.cnf.in @@ -0,0 +1,4 @@ +[client] +user=root +host=localhost +password={ DB_ROOT_PASSWORD }