From 5685092b57df474a01443ad3bb957323769fa360 Mon Sep 17 00:00:00 2001 From: Martin Kysel Date: Tue, 3 Dec 2024 13:10:16 -0500 Subject: [PATCH 1/5] Split off the sync service --- .../templates/{deployment.yaml => api.yaml} | 6 +- helm/xmtpd/templates/sync.yaml | 56 +++++++++++++++++++ test/testlib/xmtp_xmtpd.go | 18 ++++-- 3 files changed, 69 insertions(+), 11 deletions(-) rename helm/xmtpd/templates/{deployment.yaml => api.yaml} (92%) create mode 100644 helm/xmtpd/templates/sync.yaml diff --git a/helm/xmtpd/templates/deployment.yaml b/helm/xmtpd/templates/api.yaml similarity index 92% rename from helm/xmtpd/templates/deployment.yaml rename to helm/xmtpd/templates/api.yaml index 4a83369..e8f3c0e 100644 --- a/helm/xmtpd/templates/deployment.yaml +++ b/helm/xmtpd/templates/api.yaml @@ -1,7 +1,7 @@ apiVersion: apps/v1 kind: Deployment metadata: - name: {{ include "xmtpd.fullname" . }} + name: {{ include "xmtpd.fullname" . }}-api labels: {{- include "xmtpd.labels" . | nindent 4 }} spec: @@ -37,10 +37,6 @@ spec: env: - name: XMTPD_REPLICATION_ENABLE value: "true" - - name: XMTPD_SYNC_ENABLE - value: "true" - - name: XMTPD_INDEXER_ENABLE - value: "true" {{- include "helpers.list-env-variables" . | indent 12 }} ports: - name: grpc diff --git a/helm/xmtpd/templates/sync.yaml b/helm/xmtpd/templates/sync.yaml new file mode 100644 index 0000000..68389ec --- /dev/null +++ b/helm/xmtpd/templates/sync.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "xmtpd.fullname" . }}-sync + labels: + {{- include "xmtpd.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "xmtpd.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "xmtpd.labels" . | nindent 8 }} + {{- with .Values.podLabels }} + {{- toYaml . | nindent 8 }} + {{- end }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "xmtpd.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + - name: XMTPD_SYNC_ENABLE + value: "true" + - name: XMTPD_INDEXER_ENABLE + value: "true" + {{- include "helpers.list-env-variables" . | indent 12 }} + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/test/testlib/xmtp_xmtpd.go b/test/testlib/xmtp_xmtpd.go index 315fad6..953f3bb 100644 --- a/test/testlib/xmtp_xmtpd.go +++ b/test/testlib/xmtp_xmtpd.go @@ -64,21 +64,26 @@ func startXMTPDTemplate(t *testing.T, options *helm.Options, replicaCount int, n return } - xmtpdDeployment := helmChartReleaseName + xmtpdDeploymentSync := fmt.Sprintf("%s-sync", helmChartReleaseName) + xmtpdDeploymentApi := fmt.Sprintf("%s-api", helmChartReleaseName) defer func() { // collect some useful diagnostics if t.Failed() { // ignore any errors. This is already failed - _ = k8s.RunKubectlE(t, kubectlOptions, "describe", "deployment", xmtpdDeployment) + _ = k8s.RunKubectlE(t, kubectlOptions, "describe", "deployment", xmtpdDeploymentSync) + _ = k8s.RunKubectlE(t, kubectlOptions, "describe", "deployment", xmtpdDeploymentApi) } }() - AwaitNrReplicasScheduled(t, namespaceName, xmtpdDeployment, replicaCount) + AwaitNrReplicasScheduled(t, namespaceName, xmtpdDeploymentSync, replicaCount) + AwaitNrReplicasScheduled(t, namespaceName, xmtpdDeploymentApi, replicaCount) - pods := FindPodsFromChart(t, namespaceName, xmtpdDeployment) + podsSync := FindPodsFromChart(t, namespaceName, xmtpdDeploymentSync) + podsApi := FindPodsFromChart(t, namespaceName, xmtpdDeploymentApi) - for _, pod := range pods { + allPods := append(podsSync, podsApi...) + for _, pod := range allPods { AddTeardown(TEARDOWN_XMTPD, func() { if t.Failed() { // dump diagnostic info to test logs @@ -90,7 +95,8 @@ func startXMTPDTemplate(t *testing.T, options *helm.Options, replicaCount int, n }) } - AwaitNrReplicasReady(t, namespaceName, xmtpdDeployment, replicaCount) + AwaitNrReplicasReady(t, namespaceName, xmtpdDeploymentSync, replicaCount) + AwaitNrReplicasReady(t, namespaceName, xmtpdDeploymentApi, replicaCount) return } From 972c3af3e4d1caf07e7cca31c521bb3125ba9a14 Mon Sep 17 00:00:00 2001 From: Martin Kysel Date: Tue, 3 Dec 2024 19:56:12 -0500 Subject: [PATCH 2/5] missed review comment --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index e7ea867..3ee9a82 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,7 +22,7 @@ jobs: go-lint-fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Go uses: actions/setup-go@v2 From 287f2190ec88f380317eee6dd2768e9cebd27f25 Mon Sep 17 00:00:00 2001 From: Martin Kysel Date: Wed, 4 Dec 2024 13:39:40 -0500 Subject: [PATCH 3/5] healthcheck --- helm/xmtpd/templates/api.yaml | 4 ++++ helm/xmtpd/templates/sync.yaml | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/helm/xmtpd/templates/api.yaml b/helm/xmtpd/templates/api.yaml index e8f3c0e..abd2754 100644 --- a/helm/xmtpd/templates/api.yaml +++ b/helm/xmtpd/templates/api.yaml @@ -37,6 +37,10 @@ spec: env: - name: XMTPD_REPLICATION_ENABLE value: "true" + - name: XMTPD_METRICS_ENABLE + value: "true" + - name: XMTPD_METRICS_METRICS_ADDRESS + value: "0.0.0.0" {{- include "helpers.list-env-variables" . | indent 12 }} ports: - name: grpc diff --git a/helm/xmtpd/templates/sync.yaml b/helm/xmtpd/templates/sync.yaml index 68389ec..e1b87bc 100644 --- a/helm/xmtpd/templates/sync.yaml +++ b/helm/xmtpd/templates/sync.yaml @@ -39,7 +39,25 @@ spec: value: "true" - name: XMTPD_INDEXER_ENABLE value: "true" + - name: XMTPD_METRICS_ENABLE + value: "true" + - name: XMTPD_METRICS_METRICS_ADDRESS + value: "0.0.0.0" {{- include "helpers.list-env-variables" . | indent 12 }} + readinessProbe: + httpGet: + path: /metrics + port: 8008 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /metrics + port: 8008 + scheme: HTTP + failureThreshold: 3 + periodSeconds: 10 resources: {{- toYaml .Values.resources | nindent 12 }} {{- with .Values.nodeSelector }} From a1c845dcd7f2b196fafdd2547624f0be72011202 Mon Sep 17 00:00:00 2001 From: Martin Kysel Date: Wed, 4 Dec 2024 13:42:36 -0500 Subject: [PATCH 4/5] fix docs --- helm/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/helm/README.md b/helm/README.md index 39dbc49..5aa1083 100644 --- a/helm/README.md +++ b/helm/README.md @@ -62,15 +62,17 @@ helm install xmtpd xmtpd/ -f xmtpd.yaml ## Validating the installation -Once you have successfully installed all charts, including a DB, you should see 4 pods running. +Once you have successfully installed all charts, including a DB, you should see 6 pods running. You can confirm via: ```bash $ kubectl get pods NAME READY STATUS RESTARTS AGE mls-validation-service-75b6b96f79-kp4jq 1/1 Running 0 6h30m pg-postgresql-0 1/1 Running 0 6h26m -xmtpd-7dd49f6b88-mfwls 1/1 Running 0 4h56m -xmtpd-7dd49f6b88-xdk6k 1/1 Running 0 4h56m +xmtpd-api-7dd49f6b88-mfwls 1/1 Running 0 4h56m +xmtpd-api-7dd49f6b88-xdk6k 1/1 Running 0 4h56m +xmtpd-sync-7dd49f6b88-dn28d 1/1 Running 0 4h56m +xmtpd-sync-7dd49f6b88-39axn 1/1 Running 0 4h56m ``` ## XMTP Payer Helm Chart Installation From 2530034ec9f46189a739c1ba80b2520efc1f4aa1 Mon Sep 17 00:00:00 2001 From: Martin Kysel Date: Wed, 4 Dec 2024 14:15:13 -0500 Subject: [PATCH 5/5] reduce replicas default to 1 --- helm/README.md | 4 +--- helm/xmtpd/values.yaml | 2 +- test/xmtp-helm/base_xmtpd_test.go | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/helm/README.md b/helm/README.md index 5aa1083..9d971c7 100644 --- a/helm/README.md +++ b/helm/README.md @@ -62,7 +62,7 @@ helm install xmtpd xmtpd/ -f xmtpd.yaml ## Validating the installation -Once you have successfully installed all charts, including a DB, you should see 6 pods running. +Once you have successfully installed all charts, including a DB, you should see 4 pods running. You can confirm via: ```bash $ kubectl get pods @@ -70,9 +70,7 @@ NAME READY STATUS RESTARTS AGE mls-validation-service-75b6b96f79-kp4jq 1/1 Running 0 6h30m pg-postgresql-0 1/1 Running 0 6h26m xmtpd-api-7dd49f6b88-mfwls 1/1 Running 0 4h56m -xmtpd-api-7dd49f6b88-xdk6k 1/1 Running 0 4h56m xmtpd-sync-7dd49f6b88-dn28d 1/1 Running 0 4h56m -xmtpd-sync-7dd49f6b88-39axn 1/1 Running 0 4h56m ``` ## XMTP Payer Helm Chart Installation diff --git a/helm/xmtpd/values.yaml b/helm/xmtpd/values.yaml index e6ffa33..021011a 100644 --- a/helm/xmtpd/values.yaml +++ b/helm/xmtpd/values.yaml @@ -1,7 +1,7 @@ # Default values for xmtpd. # This is a YAML-formatted file. # Declare variables to be passed into your templates. -replicaCount: 2 +replicaCount: 1 # This sets the container image more information can be found here: https://kubernetes.io/docs/concepts/containers/images/ image: diff --git a/test/xmtp-helm/base_xmtpd_test.go b/test/xmtp-helm/base_xmtpd_test.go index 3ef19ae..23544c6 100644 --- a/test/xmtp-helm/base_xmtpd_test.go +++ b/test/xmtp-helm/base_xmtpd_test.go @@ -32,5 +32,5 @@ func TestKubernetesBasicXMTPDInstall(t *testing.T) { } defer testlib.Teardown(testlib.TEARDOWN_XMTPD) - testlib.StartXMTPD(t, &options, 2, namespace) + testlib.StartXMTPD(t, &options, 1, namespace) }