From b544442d655945b50deb0512876796d16f467c55 Mon Sep 17 00:00:00 2001 From: thr Date: Thu, 30 Nov 2023 09:16:31 +0100 Subject: [PATCH 1/9] update content with https://github.com/SoftwareAG/universalmessaging-prometheus-jmx-exporter-config --- .../helm/files/jmx_exporter.yaml | 568 +++++++++++------- 1 file changed, 339 insertions(+), 229 deletions(-) diff --git a/universalmessaging/helm/files/jmx_exporter.yaml b/universalmessaging/helm/files/jmx_exporter.yaml index d4d64fc..9363df0 100644 --- a/universalmessaging/helm/files/jmx_exporter.yaml +++ b/universalmessaging/helm/files/jmx_exporter.yaml @@ -1,265 +1,375 @@ --- - startDelaySeconds: 0 - - rules: - #Server Statistics: +startDelaySeconds: 10 + +#Prometheus agent is querying and shipping additional metrics from UM server even if we have not specified them into the rules. +#This may cause CPU spikes. We are introducing a blacklist and whitelist to limit this behaviour. +#Extend or narrow them down for your use case. +blacklistObjectNames: + - "com.softwareag.um.server:type=Broker,subtype=Configuration,*" + - "com.softwareag.um.server:type=Broker,configType=Properties,*" + - "com.softwareag.um.server:type=Broker,destinationType=Security,*" + - "com.softwareag.um.server:type=Broker,entry=acl,*" + - "com.softwareag.um.server:type=Broker,brokerName=*,interface=*" + - "com.softwareag.um.server:type=Broker,brokerName=*,destinationType=Topic,destinationName=*,subtype=Configuration,*" + - "com.softwareag.um.server:type=Broker,brokerName=*,destinationType=Topic,destinationName=*,securityType=Security,entry=acl,*" + - "com.softwareag.um.server:type=Broker,brokerName=*,destinationType=Queue,destinationName=*,subtype=Configuration,*" + - "com.softwareag.um.server:type=Broker,brokerName=*,destinationType=Queue,destinationName=*,securityType=Security,entry=acl,*" + +whitelistObjectNames: + - "com.softwareag.um.server:type=Broker,brokerName=*" + - "com.softwareag.um.server:type=Broker,brokerName=*,destinationType=Queue,*" + - "com.softwareag.um.server:type=Broker,brokerName=*,destinationType=Topic,*" + - "com.softwareag.um.server:type=Broker,brokerName=*,threadPool=ThreadPoolMonitor,threadPoolName=*" + - "com.softwareag.um.server:type=Broker,brokerName=*,clusterType=Cluster,clusterName=*" + - "com.softwareag.um.server:type=Broker,brokerName=*,session=Sessions,*" + +rules: + #Server Statistics #Heap Memory Usage - HeapMemory - pattern: com.softwareag.um.server<>(HeapMemory) - name: HeapMemory - help: "Server Statistics" - type: COUNTER - - #Heap Memory free - MemoryFree + name: sag_um_server_memory_heap_usage_bytes + help: "Amount of heap memory used by the server in bytes" + type: GAUGE + + #Heap Memory Free - MemoryFree - pattern: com.softwareag.um.server<>(MemoryFree) - name: MemoryFree - help: "Server Statistics" - type: COUNTER - - #Direct Memory usage - DirectMemory - - pattern: com.softwareag.um.server<>(DirectMemory) - name: DirectMemory - help: "Server Statistics" - type: COUNTER - - #Direct Memory free - MemoryMax + name: sag_um_server_memory_heap_free_bytes + help: "Amount of freely available memory on the server in bytes" + type: GAUGE + + #Heap Memory Max - MemoryMax - pattern: com.softwareag.um.server<>(MemoryMax) - name: MemoryMax - help: "Server Statistics" - type: COUNTER - - #Disk usage - DiskUsage + name: sag_um_server_memory_heap_max_bytes + help: "Maximum amount of memory available for the server in bytes" + type: GAUGE + + #Direct Memory Max - DirectMemoryMax + - pattern: com.softwareag.um.server<>(DirectMemoryMax) + name: sag_um_server_memory_direct_max_bytes + help: "Maximum amount of direct memory for the server in bytes" + type: GAUGE + + #Direct Memory Usage - DirectMemoryUsage + - pattern: com.softwareag.um.server<>(DirectMemoryUsage) + name: sag_um_server_memory_direct_usage_bytes + help: "Amount of direct memory used by the server in bytes" + type: GAUGE + + #Direct Memory Free - DirectMemoryFree + - pattern: com.softwareag.um.server<>(DirectMemoryFree) + name: sag_um_server_memory_direct_free_bytes + help: "Amount of direct memory free for the server in bytes" + type: GAUGE + + #Disk Total - DiskTotal + - pattern: com.softwareag.um.server<>(DiskTotal) + name: sag_um_server_disk_total_bytes + help: "Total space on disk in bytes" + type: GAUGE + + #Disk Usage - DiskUsage - pattern: com.softwareag.um.server<>(DiskUsage) - name: DiskUsage - help: "Server Statistics" - type: COUNTER - + name: sag_um_server_disk_usage_bytes + help: "Server disk usage in bytes" + type: GAUGE + + #Disk Free - DiskFree + - pattern: com.softwareag.um.server<>(DiskFree) + name: sag_um_server_disk_free_bytes + help: "Free space on disk in bytes" + type: GAUGE + #CPU - pattern: com.softwareag.um.server<>(CPU) - name: CPU - help: "Server Statistics" - type: COUNTER - + name: sag_um_server_cpu_usage_ratio + help: "Server CPU usage ratio" + type: GAUGE + #Number Of Current Connections - CurrentConnections - pattern: com.softwareag.um.server<>(CurrentConnections) - name: CurrentConnections - help: "Server Statistics" - type: COUNTER - + name: sag_um_server_currentconnections_total + help: "Number of current client connections to the server" + type: GAUGE + #BytesIn - pattern: com.softwareag.um.server<>(BytesIn) - name: BytesIn - help: "Server Statistics" + name: sag_um_server_bytesin_bytes + help: "Total bytes received by the server" type: COUNTER - - #BytesOut + + #BytesOut - pattern: com.softwareag.um.server<>(BytesOut) - name: BytesOut - help: "Server Statistics" + name: sag_um_server_bytesout_bytes + help: "Total bytes sent by the server" type: COUNTER - + #Events Published - TotalPublishedEvents - pattern: com.softwareag.um.server<>(TotalPublishedEvents) - name: TotalPublishedEvents - help: "Server Statistics" + name: sag_um_server_publishedevents_total + help: "Total number of events published by the server" type: COUNTER - + #Events Consumed - TotalConsumedCount - pattern: com.softwareag.um.server<>(TotalConsumedCount) - name: TotalConsumedCount - help: "Server Statistics" + name: sag_um_server_consumedevents_total + help: "Total number of events consumed by the server" type: COUNTER - + + #Cluster Metrics - #QueueSize + #Process Queue Size - pattern: com.softwareag.um.server<>(ProcessQueue) - name: ProcessQueue - help: "Cluster Metrics" - type: COUNTER - - #ClientsSize + name: sag_um_cluster_processqueue_total + help: "Size of the cluster queue waiting to be processed" + type: GAUGE + + #Client Queue Size - pattern: com.softwareag.um.server<>(ClientQueue) - name: ClientQueue - help: "Cluster Metrics" - type: COUNTER - - #CommQueueSize + name: sag_um_cluster_clientqueue_total + help: "Size of the local cluster client queue" + type: GAUGE + + #Comms Queue Size - pattern: com.softwareag.um.server<>(CommsQueue) - name: CommsQueue - help: "Cluster Metrics" - type: COUNTER - - #The Cluster node state - - pattern: com.softwareag.um.server<>(CurrentState) - name: CurrentState - help: "Cluster Metrics" - type: COUNTER - # name: CurrentState - # value: 1 - - #The Cluster state - IsOnline - - pattern: com.softwareag.um.server<>(IsOnline) - name: IsOnline - help: "Cluster Metrics" - type: COUNTER - - #Thread Pools Statistics - #Thread Pool's Idle Thread + name: sag_um_cluster_commsqueue_total + help: "Size of the incoming cluster request queue" + type: GAUGE + + #The Cluster Node State + - pattern: com.softwareag.um.server<>.*CurrentState:\s(.*) + name: sag_um_cluster_currentstate_info + value: 1 + labels: + state: $4 + current_node: $1 + cluster_name: $3 + help: "Current state of the cluster node" + + + #Thread Pool Statistics + #Thread Pool's Idle Threads - pattern: com.softwareag.um.server<>(Idle Threads) - name: Idle Threads_$3 - help: "Thread Pools Statistics" - type: COUNTER - - #Thread Pool's Allocated Thread - #Thread Pool's task queue - - pattern: com.softwareag.um.server<>(Queueed Tasks) - name: Queueed Tasks_$3 - help: "Thread Pools Statistics" - type: COUNTER - - #Number of slow tasks + name: sag_um_threadpool_idle_threads + labels: + name: $3 + help: "Number of current idle threads (allocated but not processing any tasks) for the thread pool" + type: GAUGE + + #Thread Pool's Queued Tasks + - pattern: com.softwareag.um.server<>(Queued Tasks) + name: sag_um_threadpool_queued_tasks + labels: + name: $3 + help: "Number of queued tasks waiting to be executed for the thread pool" + type: GAUGE + + #Number Of Allocated Tasks - pattern: com.softwareag.um.server<>(Allocated Threads) - name: Allocated Threads_$3 - help: "Thread Pools Statistics" - type: COUNTER - - #Number of stalled tasks + name: sag_um_threadpool_allocated_threads + labels: + name: $3 + help: "Number of threads currently allocated for the thread pool" + type: GAUGE + + #Number Of Stalled Tasks - pattern: com.softwareag.um.server<>(Stalled Tasks) - name: Stalled Tasks_$3 - help: "Thread Pools Statistics" - type: COUNTER - - #Tasks Added. - - pattern: com.softwareag.um.server<>(Tasks Added) - name: Tasks Added_$3 - help: "Thread Pools Statistics" - type: COUNTER - + name: sag_um_threadpool_stalled_tasks + labels: + name: $3 + help: "Number of stalled tasks since the last check was performed for the thread pool" + type: GAUGE + + #Event Handling - #Per Store - Topic Events Published - - pattern: com.softwareag.um.server<>(TotalPublished) - name: TotalPublished_$3 - help: "Topic Event Handling" - type: COUNTER - - #Per Store - Total Consumed Events - - pattern: com.softwareag.um.server(TotalConsumed) - name: TotalConsumed_$3 - help: "Topic Event Handling" - type: COUNTER - - #Per Store: Еvents In Store - - pattern: com.softwareag.um.server<>(NoOfEvents) - name: NoOfEvents_$3 - help: "Topic Event Handling" - type: COUNTER - + #Per Store: Topic Events Published + - pattern: com.softwareag.um.server<>(TotalPublished) + name: sag_um_topic_published_total + labels: + name: $2 + help: "Total number of events published on the channel" + type: COUNTER + + #Per Store: Total Consumed Events + - pattern: com.softwareag.um.server<>(TotalConsumed) + name: sag_um_topic_consumed_total + labels: + name: $2 + help: "Total number of events consumed from the channel" + type: COUNTER + + #Per Store: Topic Events In Store + - pattern: com.softwareag.um.server<>(NoOfEvents) + name: sag_um_topic_noofevents_total + labels: + name: $2 + help: "Total number of events on the channel" + type: GAUGE + #Per Store: Memory Usage - - pattern: com.softwareag.um.server<>(MemoryUsage) - name: MemoryUsage_$3 - help: "Topic Event Handling" - type: COUNTER - - #Per Store: Disk Usage - - pattern: com.softwareag.um.server<>(UsedSpace) - name: UsedSpace_$3 - help: "Topic Event Handling" - type: COUNTER - - #Per Store: Current Connections(subscribers) - - pattern: com.softwareag.um.server<>(CurrentConnections) - name: CurrentConnections_$3 - help: "Topic Event Handling" - type: COUNTER - - #Per Durable: durable Node information - Outstanding, Durable → Total Events - - pattern: com.softwareag.um.server<>(TotalEvents) - name: Durable_TotalEvents_$4_$5 - help: "Event Handling durable" - type: COUNTER - - - pattern: com.softwareag.um.server<>(TotalEvents) - name: Durable_TotalEvents_$5_$6 - help: "Event Handling durable" - type: COUNTER - - - pattern: com.softwareag.um.server<>(Pending) - name: Durable_Pending_$4_$5 - help: "Event Handling" - type: COUNTER - - - pattern: com.softwareag.um.server<>(Pending) - name: Durable_Pending_$5_$6 - help: "Event Handling" - type: COUNTER - - #Number of Connections(Subscribers) - #Durable Connections- Per Connection Statistics + - pattern: com.softwareag.um.server<>(MemoryUsage) + name: sag_um_topic_memory_usage_bytes + labels: + name: $2 + help: "Amount of memory used by the channel in bytes" + type: GAUGE + + #Per Store: Disk Usage + - pattern: com.softwareag.um.server<>(UsedSpace) + name: sag_um_topic_disk_usage_bytes + labels: + name: $2 + help: "Current space used by the channel in bytes" + type: GAUGE + + #Per Store: Current Connections (Subscribers) + - pattern: com.softwareag.um.server<>(CurrentConnections) + name: sag_um_topic_currentconnections_total + labels: + name: $2 + help: "Current number of channel consumers" + type: GAUGE + + + #Per Durable: Durable Node Information - Outstanding, Durable Total Events + - pattern: com.softwareag.um.server<>(TotalEvents) + name: sag_um_topic_durable_outstanding_total + labels: + topic_name: $2 + durable_name: $5 + durable_type: $4 + help: "Total number of outstanding events for the durable" + type: GAUGE + + #Per Durable: Durable Node Information - Pending, Durable Pending Events + - pattern: com.softwareag.um.server<>(TotalPending) + name: sag_um_topic_durable_pending_total + labels: + topic_name: $2 + durable_name: $5 + durable_type: $4 + help: "Total number of pending unacknowledged events for the durable" + type: GAUGE + + + #Number of Connections (Subscribers) + #Durable Connections - Per Connection Statistics #Acknowledged - - pattern: com.softwareag.um.server<>(Acknowledged) - name: Durable_Acknowledged_$5_$6_$7 - help: "Acknowledged" - type: COUNTER - - - pattern: com.softwareag.um.server<>(Acknowledged) - name: Durable_Acknowledged_$6_$7_$8 - help: "Acknowledged" - type: COUNTER - + - pattern: com.softwareag.um.server<>(Acknowledged) + name: sag_um_topic_durable_connection_acknowledged_total + labels: + topic_name: $2 + durable_name: $5 + connection: $6 + help: "Number of events acknowledged by the connection" + type: COUNTER + #RolledBack - pattern: com.softwareag.um.server<>(RolledBack) - name: Durable_RolledBack_$5_$6_$7 - help: "RolledBack" - type: COUNTER - - - pattern: com.softwareag.um.server<>(RolledBack) - name: Durable_RolledBack_$6_$7_$8 - help: "RolledBack" - type: COUNTER - + name: sag_um_topic_durable_connection_rolledBack_total + labels: + topic_name: $2 + durable_name: $5 + durable_type: $4 + connection: $6 + help: "Number of events rolled back by the connection" + type: COUNTER + #Pending - pattern: com.softwareag.um.server<>(Pending) - name: Durable_Pending_$5_$6_$7 - help: "Pending" - type: COUNTER - - - pattern: com.softwareag.um.server<>(Pending) - name: Durable_Pending_$6_$7_$8 - help: "Pending" - type: COUNTER - - #Queue data - #Per Store - Queue Events Published - - pattern: com.softwareag.um.server<>(TotalPublished) - name: TotalPublished_$3 - help: "Queue data" - type: COUNTER - - #Per Store - Total Consumed Events - - pattern: com.softwareag.um.server<>(TotalConsumed) - name: TotalConsumed_$3 - help: "Queue data" - type: COUNTER - - #Per Store: Еvents In Store - - pattern: com.softwareag.um.server<>(NoOfEvents) - name: NoOfEvents_$3 - help: "Queue data" - type: COUNTER - + name: sag_um_topic_durable_connection_pending_total + labels: + topic_name: $2 + durable_name: $5 + durable_type: $4 + connection: $6 + help: "Number of events pending acknowledgement by the connection" + type: GAUGE + + + #Queue Data + #Per Store: Queue Events Published + - pattern: com.softwareag.um.server<>(TotalPublished) + name: sag_um_queue_published_total + labels: + name: $2 + help: "Total number of events published on the queue" + type: COUNTER + + #Per Store: Total Consumed Events + - pattern: com.softwareag.um.server<>(TotalConsumed) + name: sag_um_queue_consumed_total + labels: + name: $2 + help: "Total number of events consumed from the queue" + type: COUNTER + + #Per Store: Queue Events In Store + - pattern: com.softwareag.um.server<>(NoOfEvents) + name: sag_um_queue_events_total + cache: true + labels: + name: $2 + help: "Total number of events on the queue" + type: GAUGE + #Per Store: Memory Usage - - pattern: com.softwareag.um.server<>(MemoryUsage) - name: MemoryUsage_$3 - help: "Queue data" - type: COUNTER - - #Per Store: Disk Usage - - pattern: com.softwareag.um.server<>(UsedSpace) - name: UsedSpace_$3 - help: "Queue data" - type: COUNTER - - #Per Store: Current Connections(subscribers) - - pattern: com.softwareag.um.server<>(CurrentConnections) - name: CurrentConnections_$3 - help: "Queue data" - type: COUNTER \ No newline at end of file + - pattern: com.softwareag.um.server<>(MemoryUsage) + name: sag_um_queue_memory_usage_bytes + labels: + name: $2 + help: "Amount of memory used by the queue in bytes" + type: GAUGE + + #Per Store: Disk Usage + - pattern: com.softwareag.um.server<>(UsedSpace) + name: sag_um_queue_disk_usage_bytes + labels: + name: $2 + help: "Current space used by the queue in bytes" + type: GAUGE + + #Per Store: Current Connections (Subscribers) + - pattern: com.softwareag.um.server<>(CurrentConnections) + name: sag_um_queue_currentconnections_total + labels: + name: $2 + help: "Current number of queue consumers" + type: GAUGE + + + #Queue Connection Metrics + #Acknowledged + - pattern: com.softwareag.um.server<>(Acknowledged) + name: sag_um_queue_connection_acknowledged_total + labels: + name: $2 + connection: $4 + help: "Number of events acknowledged by the connection" + type: COUNTER + + #RolledBack + - pattern: com.softwareag.um.server<>(RolledBack) + name: sag_um_queue_connection_rolledBack_total + labels: + name: $2 + connection: $4 + help: "Number of events rolled back by the connection" + type: COUNTER + + #Pending + - pattern: com.softwareag.um.server<>(Pending) + name: sag_um_queue_connection_pending_total + labels: + name: $2 + connection: $4 + help: "Number of events pending acknowledgement by the connection" + type: GAUGE + + #User Connections + #NumberOfConnections + - pattern: com.softwareag.um.server<>(NumberOfConnections) + name: sag_um_connections_per_user + labels: + name: $2 + help: "Number of user connections currently connected to the server" + type: GAUGE \ No newline at end of file From e978b3e0ce7d4e7c38f867a3c5e2304511bc5d4f Mon Sep 17 00:00:00 2001 From: thr Date: Thu, 30 Nov 2023 09:23:00 +0100 Subject: [PATCH 2/9] Mount custom configuration files Custoem_server_Common.conf and JMX exporter into container. --- .../helm/templates/statefulset.yaml | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/universalmessaging/helm/templates/statefulset.yaml b/universalmessaging/helm/templates/statefulset.yaml index 5719659..fbe1323 100644 --- a/universalmessaging/helm/templates/statefulset.yaml +++ b/universalmessaging/helm/templates/statefulset.yaml @@ -57,7 +57,21 @@ spec: - name: licence-xml configMap: name: {{ .Values.licenseConfigMap }} - defaultMode: 420 + defaultMode: 420 + - name: jmx-exporter-config + configMap: + name: {{ include "common.names.fullname" . }} + defaultMode: 420 + items: + - key: jmx_exporter.yaml + path: jmx_sag_um_exporter.yaml + - name: custom-server-common + configMap: + name: {{ include "common.names.fullname" . }} + defaultMode: 420 + items: + - key: Custom_Server_Common.conf + path: Custom_Server_Common.conf {{- if .Values.extraVolumes }} {{- if eq "string" (printf "%T" .Values.extraVolumes) }} {{ tpl .Values.extraVolumes . | nindent 8 }} @@ -92,6 +106,14 @@ spec: mountPath: /opt/softwareag/common/conf - name: licence-xml mountPath: /opt/softwareag/UniversalMessaging/server/umserver/licence/ + - name: jmx-exporter-config + mountPath: /opt/softwareag/UniversalMessaging/server/umserver/bin/jmx_sag_um_exporter.yaml + subPath: jmx_sag_um_exporter.yaml + readOnly: true + - name: custom-server-common + mountPath: /opt/softwareag/UniversalMessaging/server/umserver/bin/Custom_Server_Common.conf + subPath: Custom_Server_Common.conf + readOnly: true {{- if .Values.extraVolumeMounts }} {{- if eq "string" (printf "%T" .Values.extraVolumeMounts) }} {{- tpl .Values.extraVolumeMounts . | nindent 12 }} From d3a176f2617aefd5dba07c718396cc1992057e75 Mon Sep 17 00:00:00 2001 From: thr Date: Thu, 30 Nov 2023 09:47:03 +0100 Subject: [PATCH 3/9] update --- universalmessaging/helm/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/universalmessaging/helm/values.yaml b/universalmessaging/helm/values.yaml index 841630f..e968da0 100644 --- a/universalmessaging/helm/values.yaml +++ b/universalmessaging/helm/values.yaml @@ -87,7 +87,7 @@ nameOverride: "" # -- Overwrites full workload name. As default, the workload name is release name + '-' + Chart name. fullnameOverride: "" -# -- Custom server configuration file +# -- Custom server configuration file. Overwriting the content of file `Custom_Server_Common.conf` in container. customServerConfig: content: | From 8436895c9c65bd2dd6196054b4f9b30d60b32ba6 Mon Sep 17 00:00:00 2001 From: thr Date: Thu, 30 Nov 2023 12:40:33 +0100 Subject: [PATCH 4/9] Nginx support added --- microservicesruntime/helm/Chart.yaml | 2 +- universalmessaging/helm/README.md.gotmpl | 7 ++ .../helm/templates/nginx-configmap.yaml | 83 +++++++++++++++++++ .../helm/templates/nginx-deployment.yaml | 79 ++++++++++++++++++ .../helm/templates/nginx-svc.yaml | 43 ++++++++++ universalmessaging/helm/values.yaml | 11 +++ 6 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 universalmessaging/helm/templates/nginx-configmap.yaml create mode 100644 universalmessaging/helm/templates/nginx-deployment.yaml create mode 100644 universalmessaging/helm/templates/nginx-svc.yaml diff --git a/microservicesruntime/helm/Chart.yaml b/microservicesruntime/helm/Chart.yaml index 449395d..e24de0f 100644 --- a/microservicesruntime/helm/Chart.yaml +++ b/microservicesruntime/helm/Chart.yaml @@ -25,7 +25,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 1.0.0 +version: 1.0.1 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/universalmessaging/helm/README.md.gotmpl b/universalmessaging/helm/README.md.gotmpl index b641191..ddb64f1 100644 --- a/universalmessaging/helm/README.md.gotmpl +++ b/universalmessaging/helm/README.md.gotmpl @@ -38,4 +38,11 @@ Install release helm install um webmethods/universalmessaging ``` +## Version History + +| Version | Changes and Description | +|-----|------| +| `1.0.0' | Initial release | +| `1.0.1' | Update JMX Exporter configuration file with . Bigfix: Mount configuration files into container. Nginx added. | + {{ template "chart.valuesSection" . }} diff --git a/universalmessaging/helm/templates/nginx-configmap.yaml b/universalmessaging/helm/templates/nginx-configmap.yaml new file mode 100644 index 0000000..9edf8f3 --- /dev/null +++ b/universalmessaging/helm/templates/nginx-configmap.yaml @@ -0,0 +1,83 @@ +{{- if .Values.externalLoadBalancer }} +# /* +# * Copyright (c) 2023 Software AG, Darmstadt, Germany and/or its licensors +# * +# * SPDX-License-Identifier: Apache-2.0 +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# * +# */ + +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "common.names.fullname" . }}-nginx + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- with .Values.extraLabels -}} + {{ toYaml . | nindent 4 }} + {{- end }} + +data: + nginx.conf: | + user nginx; + worker_processes auto; + pid /var/run/nginx.pid; + + #Set the number of open file descriptors configuration according to the resource limits of the machine. + worker_rlimit_nofile 25000; + + events { + #Set this configuration according to the resource limits of the machine. + worker_connections 20000; + } + + http { + #Log settings. You can customize them as required. + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + #The server closes idle connections after this timeout. + keepalive_timeout 300s; + + #The number of requests that the client can make over a single connection. + keepalive_requests 1000000; + + #Required setting because the UM client sends "User-Agent: Mozilla/4.0 (compatible; MSIE 6.0; + #Windows NT 5.0)." The Nginx default value is 'msie6', which causes Nginx to close + #UM client connections. + keepalive_disable none; + + upstream um-host { + server {{ include "common.names.fullname" . }}-0:9000; + } + + server { + listen 80; + server_name proxy_server; + + location / { + #Important: Nginx must continuously send data to the Universal Messaging + #client rather than buffering it. + proxy_buffering off; + + #Important: Configure proxy http protocol version 1.1 to enable the connection + #keepalive and specify an empty string for the connection header. + proxy_http_version 1.1; + proxy_set_header Connection ""; + proxy_pass http://um-host; + } + } + } +{{- end }} \ No newline at end of file diff --git a/universalmessaging/helm/templates/nginx-deployment.yaml b/universalmessaging/helm/templates/nginx-deployment.yaml new file mode 100644 index 0000000..d93cd72 --- /dev/null +++ b/universalmessaging/helm/templates/nginx-deployment.yaml @@ -0,0 +1,79 @@ +{{- if .Values.externalLoadBalancer }} +--- +# /* +# * Copyright (c) 2023 Software AG, Darmstadt, Germany and/or its licensors +# * +# * SPDX-License-Identifier: Apache-2.0 +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# * +# */ + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "common.names.fullname" . }}-nginx + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- with .Values.extraLabels -}} + {{ toYaml . | nindent 4 }} + {{- end }} + +spec: + replicas: 1 + selector: + matchLabels: + {{- include "common.labels.matchLabels" . | nindent 6 }} + strategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + creationTimestamp: null + labels: + app: nginx + {{- include "common.labels.standard" . | nindent 8 }} + {{- with .Values.extraLabels -}} + {{ toYaml . | nindent 8 }} + {{- end }} + spec: + containers: + - name: nginx + image: nginx + imagePullPolicy: IfNotPresent + resources: + {{- toYaml .Values.resources.nginxContainer | nindent 10 }} + ports: + - containerPort: 9000 + name: nsp + protocol: TCP + volumeMounts: + - name: nginx-config + mountPath: /etc/nginx/nginx.conf + subPath: nginx.conf + readOnly: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + + volumes: + - name: nginx-config + configMap: + name: {{ include "common.names.fullname" . }}-nginx + items: + - key: nginx.conf + path: nginx.conf +{{- end }} \ No newline at end of file diff --git a/universalmessaging/helm/templates/nginx-svc.yaml b/universalmessaging/helm/templates/nginx-svc.yaml new file mode 100644 index 0000000..7baceb2 --- /dev/null +++ b/universalmessaging/helm/templates/nginx-svc.yaml @@ -0,0 +1,43 @@ +{{- if .Values.externalLoadBalancer }} +# /* +# * Copyright (c) 2023 Software AG, Darmstadt, Germany and/or its licensors +# * +# * SPDX-License-Identifier: Apache-2.0 +# * +# * Licensed under the Apache License, Version 2.0 (the "License"); +# * you may not use this file except in compliance with the License. +# * You may obtain a copy of the License at +# * +# * http://www.apache.org/licenses/LICENSE-2.0 +# * +# * Unless required by applicable law or agreed to in writing, software +# * distributed under the License is distributed on an "AS IS" BASIS, +# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# * See the License for the specific language governing permissions and +# * limitations under the License. +# * +# */ + +apiVersion: v1 +kind: Service +metadata: + name: {{ include "common.names.fullname" . }}-nginx + labels: + {{- include "common.labels.standard" . | nindent 4 }} + {{- with .Values.extraLabels -}} + {{ toYaml . | nindent 4 }} + {{- end }} + +spec: + type: {{ .Values.service.type }} + ports: + - port: 9000 + protocol: TCP + targetPort: 9000 + name: rt-port + {{- if (eq .Values.service.type "ClusterIP") }} + nodePort: null + {{- end }} + selector: + {{- include "common.labels.matchLabels" . | nindent 4 }} +{{- end }} \ No newline at end of file diff --git a/universalmessaging/helm/values.yaml b/universalmessaging/helm/values.yaml index e968da0..4c94214 100644 --- a/universalmessaging/helm/values.yaml +++ b/universalmessaging/helm/values.yaml @@ -166,6 +166,14 @@ resources: {} # requests: # cpu: 100m # memory: 128Mi + # nginxContainer: + # requests: + # cpu: 500m + # memory: 2Gi + # limits: + # # use a high cpu limit to avaoid the container being throttled + # cpu: 8 + # memory: 4Gi nodeSelector: {} @@ -186,3 +194,6 @@ readinessProbe: timeoutSeconds: 60 successThreshold: 1 failureThreshold: 5 + +# -- Instanciate Nginx as external LB +externalLoadBalancer: false From cd1b6b2a064759cf05862bfb756c3140cac0eec2 Mon Sep 17 00:00:00 2001 From: thr Date: Thu, 30 Nov 2023 12:41:06 +0100 Subject: [PATCH 5/9] make it equal to API GW and MSR Nginx deployment --- microservicesruntime/helm/templates/nginx-deployment.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/microservicesruntime/helm/templates/nginx-deployment.yaml b/microservicesruntime/helm/templates/nginx-deployment.yaml index 0925f01..656a2ef 100644 --- a/microservicesruntime/helm/templates/nginx-deployment.yaml +++ b/microservicesruntime/helm/templates/nginx-deployment.yaml @@ -42,7 +42,8 @@ spec: metadata: creationTimestamp: null labels: - {{- include "common.labels.standard" . | nindent 8 }}-nginx + app: nginx + {{- include "common.labels.standard" . | nindent 8 }} {{- with .Values.extraLabels -}} {{ toYaml . | nindent 8 }} {{- end }} From bc44d9414f55b922cc09d880543bdd2a29a5588e Mon Sep 17 00:00:00 2001 From: thr Date: Fri, 1 Dec 2023 09:08:16 +0100 Subject: [PATCH 6/9] link to UM documentation --- universalmessaging/helm/templates/nginx-configmap.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/universalmessaging/helm/templates/nginx-configmap.yaml b/universalmessaging/helm/templates/nginx-configmap.yaml index 9edf8f3..9877bc6 100644 --- a/universalmessaging/helm/templates/nginx-configmap.yaml +++ b/universalmessaging/helm/templates/nginx-configmap.yaml @@ -30,6 +30,7 @@ metadata: data: nginx.conf: | + # Source is from https://documentation.softwareag.com/universal_messaging/num10-15/webhelp/num-webhelp/#page/num-webhelp%2Fre-configure_nginx_to_serve_http_requests.html%23 user nginx; worker_processes auto; pid /var/run/nginx.pid; From 933346e4e4a5e0aa267575fcb2950a5c0a391657 Mon Sep 17 00:00:00 2001 From: thr Date: Fri, 1 Dec 2023 10:01:47 +0100 Subject: [PATCH 7/9] docu updated --- universalmessaging/helm/values.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/universalmessaging/helm/values.yaml b/universalmessaging/helm/values.yaml index 4c94214..36bc9bd 100644 --- a/universalmessaging/helm/values.yaml +++ b/universalmessaging/helm/values.yaml @@ -196,4 +196,5 @@ readinessProbe: failureThreshold: 5 # -- Instanciate Nginx as external LB +# Nginx is configured by example from [Universal Messaging documentation](https://documentation.softwareag.com/universal_messaging/num10-15/webhelp/num-webhelp/#page/num-webhelp%2Fre-configure_nginx_to_serve_http_requests.html%23) externalLoadBalancer: false From df7fce07ecf4d441da76d8f60eb54d8ce2746571 Mon Sep 17 00:00:00 2001 From: thr Date: Tue, 5 Dec 2023 13:19:07 +0100 Subject: [PATCH 8/9] LB dispatch requests to all replicas --- universalmessaging/helm/templates/nginx-configmap.yaml | 5 ++++- universalmessaging/helm/values.yaml | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/universalmessaging/helm/templates/nginx-configmap.yaml b/universalmessaging/helm/templates/nginx-configmap.yaml index 9877bc6..8db7db2 100644 --- a/universalmessaging/helm/templates/nginx-configmap.yaml +++ b/universalmessaging/helm/templates/nginx-configmap.yaml @@ -61,7 +61,10 @@ data: keepalive_disable none; upstream um-host { - server {{ include "common.names.fullname" . }}-0:9000; + {{- $fullName := include "common.names.fullname" . -}} + {{- range $replicaIndex := until (int .Values.replicaCount) }} + server {{ $fullName }}-{{ $replicaIndex }}:9000; + {{ end }} } server { diff --git a/universalmessaging/helm/values.yaml b/universalmessaging/helm/values.yaml index 36bc9bd..01b59b2 100644 --- a/universalmessaging/helm/values.yaml +++ b/universalmessaging/helm/values.yaml @@ -195,6 +195,6 @@ readinessProbe: successThreshold: 1 failureThreshold: 5 -# -- Instanciate Nginx as external LB +# -- Instanciate Nginx as external LB. The LB will be configured to dispatch incoming requests to all `replicaCount` replicas. # Nginx is configured by example from [Universal Messaging documentation](https://documentation.softwareag.com/universal_messaging/num10-15/webhelp/num-webhelp/#page/num-webhelp%2Fre-configure_nginx_to_serve_http_requests.html%23) externalLoadBalancer: false From 33a6d0c24f13e92ddc29cebc59261665eb61c80a Mon Sep 17 00:00:00 2001 From: thr Date: Tue, 5 Dec 2023 13:29:54 +0100 Subject: [PATCH 9/9] replace Helm operation --- .github/workflows/dry-run-test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/dry-run-test.yml b/.github/workflows/dry-run-test.yml index 7e48e8a..003f466 100644 --- a/.github/workflows/dry-run-test.yml +++ b/.github/workflows/dry-run-test.yml @@ -36,20 +36,20 @@ jobs: - name: Dry-run of apigateway run: | - helm install apigw apigateway/helm --dry-run + helm template apigw apigateway/helm - name: Dry-run of developerportal run: | - helm install devportal developerportal/helm --dry-run + helm template devportal developerportal/helm - name: Dry-run of microservicesruntime run: | - helm install msr microservicesruntime/helm --dry-run + helm template msr microservicesruntime/helm - name: Dry-run of mywebmethodsserver run: | - helm install mws mywebmethodsserver/helm --dry-run + helm template mws mywebmethodsserver/helm - name: Dry-run of universalmessaging run: | - helm install um universalmessaging/helm --dry-run + helm template um universalmessaging/helm