sutopopulating solr & rmq hostnames, claimname, mn_url

DataONEorg · Nov 29, 2023 · 065e626 · 065e626
1 parent d9cda1a
commit 065e626
Show file tree

Hide file tree

Showing 5 changed files with 117 additions and 44 deletions.
diff --git a/helm/config/dataone-indexer.properties b/helm/config/dataone-indexer.properties
@@ -1,4 +1,4 @@
-index.d1node.baseURL={{ .Values.idxworker.mn_url }}
+index.d1node.baseURL={{ include "idxworker.mn.url" . }}
 index.data.root.directory={{ .Values.idxworker.data_directory }}
 index.document.root.directory={{ .Values.idxworker.document_directory }}
 index.tdb.directory={{ .Values.idxworker.tripleDbDirectory }}
@@ -12,7 +12,7 @@ index.rabbitmq.password=${env:RABBITMQ_PASSWORD}
 #If changing max priority, existing queue must be deleted, and consumers must use the same number
 index.rabbitmq.max.priority=10
 
-solr.base.uri=http://{{ .Values.solr.hostname }}:
+solr.base.uri=http://{{ include "idxworker.solr.hostname" . }}:
 {{- .Values.solr.service.ports.http }}/solr/{{ .Values.solr.customCollection }}
 solr.query.uri=${solr.base.uri}/select/
 solr.index.uri=${solr.base.uri}/update/?commit=true

diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
@@ -61,6 +61,31 @@ Create the name of the service account to use
 {{- end }}
 {{- end }}
 
+{{/*
+set MN url
+e.g. https://metacat-dev.test.dataone.org/metacat/d1/mn
+*/}}
+{{- define "idxworker.mn.url" -}}
+{{- if not .Values.idxworker.mn_url }}
+{{- printf "https://%s-metacat-hl/%s/d1/mn" .Release.Name .Values.global.metacatAppContext }}
+{{- else }}
+{{- .Values.idxworker.mn_url }}
+{{- end }}
+{{- end }}
+
+{{/*
+set Claim Name of existing PVC to use (typically the volume that is shared with metacat)
+Either use the value set in .Values.persistence.claimName, or if blank, autopopulate with
+  {podname}-metacat-{releaseName}-0 (e.g. metacatbrooke-metacat-metacatbrooke-0)
+*/}}
+{{- define "idxworker.shared.claimName" -}}
+{{- if not .Values.persistence.claimName }}
+{{- .Release.Name }}-metacat-{{- .Release.Name }}-0
+{{- else }}
+{{- .Values.persistence.claimName }}
+{{- end }}
+{{- end }}
+
 {{/*
 set RabbitMQ HostName
 */}}
@@ -71,3 +96,14 @@ set RabbitMQ HostName
 {{- .Values.rabbitmq.hostname }}
 {{- end }}
 {{- end }}
+
+{{/*
+set Solr HostName
+*/}}
+{{- define "idxworker.solr.hostname" -}}
+{{- if not .Values.solr.hostname }}
+{{- .Release.Name }}-solr-headless
+{{- else }}
+{{- .Values.solr.hostname }}
+{{- end }}
+{{- end }}
diff --git a/helm/templates/configmap.yaml b/helm/templates/configmap.yaml
@@ -2,7 +2,7 @@
 apiVersion: v1
 kind: ConfigMap
 metadata:
-  name: {{ .Release.Name }}-configfiles
+  name: {{ .Release.Name }}-indexer-configfiles
   labels:
     {{- include "idxworker.labels" . | nindent 4 }}
 data:

diff --git a/helm/templates/deployment.yaml b/helm/templates/deployment.yaml
@@ -49,7 +49,7 @@ spec:
               subPath: dataone-indexer.properties
               name: {{ .Release.Name }}-config-volume
             - mountPath: {{ .Values.persistence.mountPath }}
-              name: {{ .Values.persistence.volumeName }}
+              name: indexer-metacat-pv
               readOnly: true
             - mountPath: /etc/dataone/tdb-cache
               name: {{ .Release.Name }}-temp-tripledb-volume
@@ -94,13 +94,16 @@ spec:
             - sh
             - -c
             - >
-              URI="/solr/{{- .Values.solr.customCollection -}}
+              URI="/solr/
+              {{- required "solr.customCollection IS REQUIRED" .Values.solr.customCollection -}}
                   /admin/file?file=schema.xml&contentType=text/xml";
-              until [[ $(echo -e "GET $URI HTTP/1.1\r\nHost: {{ .Values.solr.hostname -}}
-              \r\n\r\n" | nc {{ .Values.solr.hostname }}{{ print " " -}}
-              {{ .Values.solr.service.ports.http }} | grep -c "<schema name=\"dataone") == 1 ]]; do
-              echo waiting for Solr Schema to be accessible at http://{{ .Values.solr.hostname }}:
-              {{- .Values.solr.service.ports.http }}$URI; sleep 1; done;
+              {{- $solrHost := (include "idxworker.solr.hostname" .) }}
+  {{- $solrPort := (required "solr.service.ports.http REQUIRED" .Values.solr.service.ports.http) }}
+              until [[ $(echo -e "GET $URI HTTP/1.1\r\nHost: {{ $solrHost -}}
+              \r\n\r\n" | nc {{ $solrHost }} {{ $solrPort -}}
+              | grep -c "<schema name=\"dataone") == 1 ]]; do
+              echo waiting for Solr Schema to be accessible at http://{{ $solrHost }}:
+              {{- $solrPort }}$URI; sleep 1; done;
       {{- with .Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
@@ -116,19 +119,19 @@ spec:
       volumes:
         - name: {{ .Release.Name }}-config-volume
           configMap:
-            name: {{ .Release.Name }}-configfiles
+            name: {{ .Release.Name }}-indexer-configfiles
             defaultMode: 0644
-        - name: {{ .Values.persistence.volumeName }}
+        - name:  indexer-metacat-pv
           persistentVolumeClaim:
-            claimName: {{ .Values.persistence.claimName }}
+            claimName: {{ include "idxworker.shared.claimName" . }}
             readOnly: true
         - name: {{ .Release.Name }}-temp-tripledb-volume
           ephemeral:
             volumeClaimTemplate:
               spec:
                 accessModes: [ "ReadWriteOnce" ]
-                {{- if ne (quote .Values.persistence.ephemeralVolumeStorageClass) "" }}
-                storageClassName: {{ .Values.persistence.ephemeralVolumeStorageClass }}
+                {{- if ne (quote .Values.global.ephemeralVolumeStorageClass) "" }}
+                storageClassName: {{ .Values.global.ephemeralVolumeStorageClass }}
                 {{ end -}}
                 resources:
                   requests:

diff --git a/helm/values.yaml b/helm/values.yaml
@@ -4,14 +4,24 @@
 
 global:
   solrPort: &global-solr-port 8983
+  metacatAppContext: metacat
 
   ## @param global.storageClass default name of the storageClass to use for PVs
   ## Comment out to use default, if one is set on your cluster
   ##
   ## To inspect your cluster to see what storageClass names are supported:
   ##    $  kubectl get storageclass
   ## (e.g. for Rancher Desktop, use:   storageClass: local-path)
-  storageClass: csi-rbd-sc
+  storageClass: csi-cephfs-sc
+
+  ## @param global.ephemeralVolumeStorageClass Optional override of global.storageClass.
+  ## Can be used to assign a storageClass that has a 'Delete' Reclaim Policy, thus allowing
+  ## ephemeral volumes to be cleaned up automatically (eg "csi-cephfs-sc-ephemeral")
+  ##
+  ephemeralVolumeStorageClass: csi-cephfs-sc-ephemeral
+
+
+## @section Dataone-Indexer Application-Specific Properties
 
 image:
   repository: ghcr.io/dataoneorg/dataone-index-worker
@@ -69,56 +79,56 @@ tolerations: []
 affinity: {}
 
 persistence:
-  claimName: d1index-metacat-pvc
-  volumeName: d1index-metacat-pv
-  mountPath: /var/metacat
-  ## @param persistence.ephemeralVolumeStorageClass Optional override of global.storageClass.
-  ## Can be used to assign a storageClass that has a `Delete` Reclaim Policy, thus allowing
-  ## ephemeral volumes to be cleaned up automatically (eg "csi-cephfs-sc-ephemeral")
+  ## @param persistence.claimName Name of existing PVC to use (typically shared with metacat)
+  ## Set a value for 'claimName' only if you want to re-use a Persistent Volume Claim that has
+  ## already been set up by a k8s admin ahead of time.
+  ## Leaving it blank will cause the value to be autopopulated with:
+  ## claimName: {podname}-metacat-{releaseName}-metacat-0
   ##
-  ephemeralVolumeStorageClass: ""
+  claimName: ""
+  mountPath: /var/metacat
+
 
-# Values for the IndexWorker
+## @section IndexWorker properties
+##
 idxworker:
+  debug: "FALSE"
+
   ## @param idxworker.mn_url URL of the metacat instance that depends upon this indexer
+  ## Leave this value unset (mn_url: "") to have it automatically populated
   ##
-  mn_url: "https://valley.duckdns.org/metacat/d1/mn"
+  mn_url: ""
 
   ## @param idxworker.cn_url URL of the CN
   ##
   cn_url: "https://cn.dataone.org/cn"
 
-  ## Location of docs/data within the metacat shared volume
+  ## @param idxworker.data_directory Location of data within the metacat shared volume
+  ##
   data_directory: /var/metacat/data
+
+  ## @param idxworker.document_directory Location of docs within the metacat shared volume
+  ##
   document_directory: /var/metacat/documents
+
   # The size of the thread pool which processes the index tasks
   pool_size: 5
-  debug: "FALSE"
   d1_serviceType_url: https://cn.dataone.org/mnServiceTypes.xml
 
   ## @param idxworker.tripleDbDirectory path to indexer cache for triples (usually on a mount)
   ##
   tripleDbDirectory: /etc/dataone/tdb-cache
 
-# The section for the rabbitMQ configuration
+## @section RabbitMQ Configuration
+##
 rabbitmq:
   enabled: true
-  auth:
-    ## @param rabbitmq.username the username for rabbitmq access
-    ##
-    username: rmq
-
-    ## @param rabbitmq.existingPasswordSecret the k8s secret holding the rabbitmq password
-    ## (must be associated with key: `rabbitmq-password`)
-    ##
-    existingPasswordSecret: ""
-
-  ## @param rabbitmq.hostname idxworker param: hostname of the rabbitmq service to use
-  ## Leave this value unset (`hostname: ""`) to have it automatically populated
+  ## @param rabbitmq.hostname (idxworker-specific param): hostname of the rabbitmq service to use
+  ## Leave this value unset (hostname: "") to have it automatically populated
   ##
   hostname: ""
-  ## @param rabbitmq.hostport idxworker param: hostport of the rabbitmq service
-  ## Leave this value unset (`hostport: ""`) to have it automatically populated
+  ## @param rabbitmq.hostport (idxworker-specific param): hostport of the rabbitmq service
+  ## Leave this value unset (hostport: "") to have it automatically populated
   ##
   hostport: ""
 
@@ -129,15 +139,39 @@ rabbitmq:
   # must use the same number.
   #max.priority: 10
 
+  auth:
+    ## @param rabbitmq.username the username for rabbitmq access
+    ##
+    username: rmq
+
+    ## @param rabbitmq.existingPasswordSecret the k8s secret holding the rabbitmq password
+    ## (must be associated with key: 'rabbitmq-password')
+    ##
+    existingPasswordSecret: ""
+
+
+## @section Solr Configuration
+##
 solr:
   collection: temp_collection
+
+  ## @param solr.customCollection (required) name of the solr collection to use
+  ##
   customCollection: dataone_index
+
   coreNames:
     - dataone_core
   #javaMem: "-Xms512m -Xmx2g"
-  hostname: d1index-solr-headless.d1index.svc
+
+  ## @param solr.hostname (idxworker-specific param): hostname of the solr service to use
+  ## Leave this value unset (hostname: "") to have it automatically populated
+  ##
+  hostname: ""
+
   service:
     ports:
+      ## @param solr.service.ports.http: see global.solrPort (required)
+      ##
       http: *global-solr-port
     nodePorts:
       http: *global-solr-port
@@ -154,7 +188,7 @@ solr:
   extraVolumes:
     - name: solr-config
       configMap:
-        name: d1index-configfiles
+        name: d1index-indexer-configfiles
         defaultMode: 0777
   extraVolumeMounts:
     - name: solr-config