diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b84aabd..891ee129 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Changed +- Upgrade cilium to `1.15.1`. + +### Changed + - Replace `null` values in `values.yaml` with its actual defaults. Config values with `null` types in the values schema prevented users from changing its values. ## [0.19.2] - 2024-01-22 diff --git a/helm/cilium/Chart.yaml b/helm/cilium/Chart.yaml index f850e5d8..3c04df82 100644 --- a/helm/cilium/Chart.yaml +++ b/helm/cilium/Chart.yaml @@ -2,10 +2,10 @@ apiVersion: v2 name: cilium displayName: Cilium home: https://cilium.io/ -version: 0.19.2 -appVersion: 1.14.5 +version: 0.20.0 +appVersion: 1.15.1 kubeVersion: ">= 1.16.0-0" -icon: https://cdn.jsdelivr.net/gh/cilium/cilium@v1.14/Documentation/images/logo-solo.svg +icon: https://cdn.jsdelivr.net/gh/cilium/cilium@v1.15/Documentation/images/logo-solo.svg description: eBPF-based Networking, Security, and Observability keywords: - BPF @@ -75,7 +75,7 @@ annotations: description: | Cilium Endpoint represents the status of individual pods or nodes in the cluster which are managed by Cilium, including enforcement status, - IP addressing and whether the networking is succesfully operational. + IP addressing and whether the networking is successfully operational. - kind: CiliumEndpointSlice version: v2alpha1 name: ciliumendpointslices.cilium.io @@ -83,7 +83,7 @@ annotations: description: | Cilium Endpoint Slice represents the status of groups of pods or nodes in the cluster which are managed by Cilium, including enforcement status, - IP addressing and whether the networking is succesfully operational. + IP addressing and whether the networking is successfully operational. - kind: CiliumEgressGatewayPolicy version: v2 name: ciliumegressgatewaypolicies.cilium.io @@ -112,6 +112,40 @@ annotations: description: | Cilium BGP Peering Policy instructs Cilium to create specific BGP peering configurations. + - kind: CiliumBGPClusterConfig + version: v2alpha1 + name: ciliumbgpclusterconfigs.cilium.io + displayName: Cilium BGP Cluster Config + description: | + Cilium BGP Cluster Config instructs Cilium operator to create specific BGP cluster + configurations. + - kind: CiliumBGPPeerConfig + version: v2alpha1 + name: ciliumbgppeerconfigs.cilium.io + displayName: Cilium BGP Peer Config + description: | + CiliumBGPPeerConfig is a common set of BGP peer configurations. It can be referenced + by multiple peers from CiliumBGPClusterConfig. + - kind: CiliumBGPAdvertisement + version: v2alpha1 + name: ciliumbgpadvertisements.cilium.io + displayName: Cilium BGP Advertisement + description: | + CiliumBGPAdvertisement is used to define source of BGP advertisement as well as BGP attributes + to be advertised with those prefixes. + - kind: CiliumBGPNodeConfig + version: v2alpha1 + name: ciliumbgpnodeconfigs.cilium.io + displayName: Cilium BGP Node Config + description: | + CiliumBGPNodeConfig is read only node specific BGP configuration. It is constructed by Cilium operator. + It will also contain node local BGP state information. + - kind: CiliumBGPNodeConfigOverride + version: v2alpha1 + name: ciliumbgpnodeconfigoverrides.cilium.io + displayName: Cilium BGP Node Config Override + description: | + CiliumBGPNodeConfigOverride can be used to override node specific BGP configuration. - kind: CiliumLoadBalancerIPPool version: v2alpha1 name: ciliumloadbalancerippools.cilium.io diff --git a/helm/cilium/README.md b/helm/cilium/README.md index 8aff3441..cd5997a5 100644 --- a/helm/cilium/README.md +++ b/helm/cilium/README.md @@ -1,6 +1,6 @@ # cilium -![Version: 1.14.5](https://img.shields.io/badge/Version-1.14.5-informational?style=flat-square) ![AppVersion: 1.14.5](https://img.shields.io/badge/AppVersion-1.14.5-informational?style=flat-square) +![Version: 1.15.1](https://img.shields.io/badge/Version-1.15.1-informational?style=flat-square) ![AppVersion: 1.15.1](https://img.shields.io/badge/AppVersion-1.15.1-informational?style=flat-square) Cilium is open source software for providing and transparently securing network connectivity and loadbalancing between application workloads such as @@ -60,24 +60,30 @@ contributors across the globe, there is almost always someone available to help. | aksbyocni.enabled | bool | `false` | Enable AKS BYOCNI integration. Note that this is incompatible with AKS clusters not created in BYOCNI mode: use Azure integration (`azure.enabled`) instead. | | alibabacloud.enabled | bool | `false` | Enable AlibabaCloud ENI integration | | annotateK8sNode | bool | `false` | Annotate k8s node upon initialization with Cilium's metadata. | +| annotations | object | `{}` | Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) | +| apiRateLimit | string | `""` | The api-rate-limit option can be used to overwrite individual settings of the default configuration for rate limiting calls to the Cilium Agent API | | authentication.enabled | bool | `true` | Enable authentication processing and garbage collection. Note that if disabled, policy enforcement will still block requests that require authentication. But the resulting authentication requests for these requests will not be processed, therefore the requests not be allowed. | | authentication.gcInterval | string | `"5m0s"` | Interval for garbage collection of auth map entries. | +| authentication.mutual.connectTimeout | string | `"5s"` | Timeout for connecting to the remote node TCP socket | | authentication.mutual.port | int | `4250` | Port on the agent where mutual authentication handshakes between agents will be performed | | authentication.mutual.spire.adminSocketPath | string | `"/run/spire/sockets/admin.sock"` | SPIRE socket path where the SPIRE delegated api agent is listening | | authentication.mutual.spire.agentSocketPath | string | `"/run/spire/sockets/agent/agent.sock"` | SPIRE socket path where the SPIRE workload agent is listening. Applies to both the Cilium Agent and Operator | +| authentication.mutual.spire.annotations | object | `{}` | Annotations to be added to all top-level spire objects (resources under templates/spire) | | authentication.mutual.spire.connectionTimeout | string | `"30s"` | SPIRE connection timeout | | authentication.mutual.spire.enabled | bool | `false` | Enable SPIRE integration (beta) | | authentication.mutual.spire.install.agent.affinity | object | `{}` | SPIRE agent affinity configuration | | authentication.mutual.spire.install.agent.annotations | object | `{}` | SPIRE agent annotations | -| authentication.mutual.spire.install.agent.image | string | `"ghcr.io/spiffe/spire-agent:1.6.3@sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823"` | SPIRE agent image | +| authentication.mutual.spire.install.agent.image | object | `{"digest":"sha256:99405637647968245ff9fe215f8bd2bd0ea9807be9725f8bf19fe1b21471e52b","override":"","pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-agent","tag":"1.8.5","useDigest":true}` | SPIRE agent image | | authentication.mutual.spire.install.agent.labels | object | `{}` | SPIRE agent labels | | authentication.mutual.spire.install.agent.nodeSelector | object | `{}` | SPIRE agent nodeSelector configuration ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | authentication.mutual.spire.install.agent.podSecurityContext | object | `{}` | Security context to be added to spire agent pods. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | | authentication.mutual.spire.install.agent.securityContext | object | `{}` | Security context to be added to spire agent containers. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container | | authentication.mutual.spire.install.agent.serviceAccount | object | `{"create":true,"name":"spire-agent"}` | SPIRE agent service account | | authentication.mutual.spire.install.agent.skipKubeletVerification | bool | `true` | SPIRE Workload Attestor kubelet verification. | -| authentication.mutual.spire.install.agent.tolerations | list | `[]` | SPIRE agent tolerations configuration ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | +| authentication.mutual.spire.install.agent.tolerations | list | `[{"effect":"NoSchedule","key":"node.kubernetes.io/not-ready"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/master"},{"effect":"NoSchedule","key":"node-role.kubernetes.io/control-plane"},{"effect":"NoSchedule","key":"node.cloudprovider.kubernetes.io/uninitialized","value":"true"},{"key":"CriticalAddonsOnly","operator":"Exists"}]` | SPIRE agent tolerations configuration By default it follows the same tolerations as the agent itself to allow the Cilium agent on this node to connect to SPIRE. ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | authentication.mutual.spire.install.enabled | bool | `true` | Enable SPIRE installation. This will only take effect only if authentication.mutual.spire.enabled is true | +| authentication.mutual.spire.install.existingNamespace | bool | `false` | SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. | +| authentication.mutual.spire.install.initImage | object | `{"digest":"sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b","override":"","pullPolicy":"IfNotPresent","repository":"docker.io/library/busybox","tag":"1.36.1","useDigest":true}` | init container image of SPIRE agent and server | | authentication.mutual.spire.install.namespace | string | `"cilium-spire"` | SPIRE namespace to install into | | authentication.mutual.spire.install.server.affinity | object | `{}` | SPIRE server affinity configuration | | authentication.mutual.spire.install.server.annotations | object | `{}` | SPIRE server annotations | @@ -87,10 +93,12 @@ contributors across the globe, there is almost always someone available to help. | authentication.mutual.spire.install.server.dataStorage.enabled | bool | `true` | Enable SPIRE server data storage | | authentication.mutual.spire.install.server.dataStorage.size | string | `"1Gi"` | Size of the SPIRE server data storage | | authentication.mutual.spire.install.server.dataStorage.storageClass | string | `""` | StorageClass of the SPIRE server data storage | -| authentication.mutual.spire.install.server.image | string | `"ghcr.io/spiffe/spire-server:1.6.3@sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f"` | SPIRE server image | +| authentication.mutual.spire.install.server.image | object | `{"digest":"sha256:28269265882048dcf0fed32fe47663cd98613727210b8d1a55618826f9bf5428","override":"","pullPolicy":"IfNotPresent","repository":"ghcr.io/spiffe/spire-server","tag":"1.8.5","useDigest":true}` | SPIRE server image | | authentication.mutual.spire.install.server.initContainers | list | `[]` | SPIRE server init containers | | authentication.mutual.spire.install.server.labels | object | `{}` | SPIRE server labels | | authentication.mutual.spire.install.server.nodeSelector | object | `{}` | SPIRE server nodeSelector configuration ref: ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | +| authentication.mutual.spire.install.server.podSecurityContext | object | `{}` | Security context to be added to spire server pods. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod | +| authentication.mutual.spire.install.server.securityContext | object | `{}` | Security context to be added to spire server containers. SecurityContext holds pod-level security attributes and common container settings. ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container | | authentication.mutual.spire.install.server.service.annotations | object | `{}` | Annotations to be added to the SPIRE server service | | authentication.mutual.spire.install.server.service.labels | object | `{}` | Labels to be added to the SPIRE server service | | authentication.mutual.spire.install.server.service.type | string | `"ClusterIP"` | Service type for the SPIRE server service | @@ -109,8 +117,11 @@ contributors across the globe, there is almost always someone available to help. | bgp.announce.loadbalancerIP | bool | `false` | Enable allocation and announcement of service LoadBalancer IPs | | bgp.announce.podCIDR | bool | `false` | Enable announcement of node pod CIDR | | bgp.enabled | bool | `false` | Enable BGP support inside Cilium; embeds a new ConfigMap for BGP inside cilium-agent and cilium-operator | -| bgpControlPlane | object | `{"enabled":false}` | This feature set enables virtual BGP routers to be created via CiliumBGPPeeringPolicy CRDs. | +| bgpControlPlane | object | `{"enabled":false,"secretsNamespace":{"create":false,"name":"kube-system"}}` | This feature set enables virtual BGP routers to be created via CiliumBGPPeeringPolicy CRDs. | | bgpControlPlane.enabled | bool | `false` | Enables the BGP control plane. | +| bgpControlPlane.secretsNamespace | object | `{"create":false,"name":"kube-system"}` | SecretsNamespace is the namespace which BGP support will retrieve secrets from. | +| bgpControlPlane.secretsNamespace.create | bool | `false` | Create secrets namespace for BGP secrets. | +| bgpControlPlane.secretsNamespace.name | string | `"kube-system"` | The name of the secret namespace to which Cilium agents are given read access | | bpf.authMapMax | int | `524288` | Configure the maximum number of entries in auth map. | | bpf.autoMount.enabled | bool | `true` | Enable automatic mount of BPF filesystem When `autoMount` is enabled, the BPF filesystem is mounted at `bpf.root` path on the underlying host and inside the cilium agent pod. If users disable `autoMount`, it's expected that users have mounted bpffs filesystem at the specified `bpf.root` volume, and then the volume will be mounted inside the cilium agent pod at the same path. | | bpf.ctAnyMax | int | `262144` | Configure the maximum number of entries for the non-TCP connection tracking table. | @@ -131,7 +142,8 @@ contributors across the globe, there is almost always someone available to help. | bpf.tproxy | bool | `false` | Configure the eBPF-based TPROXY to reduce reliance on iptables rules for implementing Layer 7 policy. | | bpf.vlanBypass | list | `[]` | Configure explicitly allowed VLAN id's for bpf logic bypass. [0] will allow all VLAN id's without any filtering. | | bpfClockProbe | bool | `false` | Enable BPF clock source probing for more efficient tick retrieval. | -| certgen | object | `{"annotations":{"cronJob":{},"job":{}},"extraVolumeMounts":[],"extraVolumes":[],"image":{"override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-certgen","tag":"v0.1.9@sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f"},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen | object | `{"affinity":{},"annotations":{"cronJob":{},"job":{}},"extraVolumeMounts":[],"extraVolumes":[],"image":{"override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-certgen","tag":"v0.1.9@sha256:89a0847753686444daabde9474b48340993bd19c7bea66a46e45b2974b82041f"},"podLabels":{},"tolerations":[],"ttlSecondsAfterFinished":1800}` | Configure certificate generation for Hubble integration. If hubble.tls.auto.method=cronJob, these values are used for the Kubernetes CronJob which will be scheduled regularly to (re)generate any certificates not provided manually. | +| certgen.affinity | object | `{}` | Affinity for certgen | | certgen.annotations | object | `{"cronJob":{},"job":{}}` | Annotations to be added to the hubble-certgen initial Job and CronJob | | certgen.extraVolumeMounts | list | `[]` | Additional certgen volumeMounts. | | certgen.extraVolumes | list | `[]` | Additional certgen volumes. | @@ -147,25 +159,29 @@ contributors across the globe, there is almost always someone available to help. | cleanupKubeProxy | bool | `false` | | | cluster.id | int | `0` | Unique ID of the cluster. Must be unique across all connected clusters and in the range of 1 to 255. Only required for Cluster Mesh, may be 0 if Cluster Mesh is not used. | | cluster.name | string | `"default"` | Name of the cluster. Only required for Cluster Mesh and mutual authentication with SPIRE. | +| clustermesh.annotations | object | `{}` | Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) | | clustermesh.apiserver.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"clustermesh-apiserver"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for clustermesh.apiserver | -| clustermesh.apiserver.etcd.image | object | `{"override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/etcd","tag":"v3.5.4@sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3"}` | Clustermesh API server etcd image. | +| clustermesh.apiserver.etcd.init.extraArgs | list | `[]` | Additional arguments to `clustermesh-apiserver etcdinit`. | +| clustermesh.apiserver.etcd.init.extraEnv | list | `[]` | Additional environment variables to `clustermesh-apiserver etcdinit`. | | clustermesh.apiserver.etcd.init.resources | object | `{}` | Specifies the resources for etcd init container in the apiserver | +| clustermesh.apiserver.etcd.lifecycle | object | `{}` | lifecycle setting for the etcd container | | clustermesh.apiserver.etcd.resources | object | `{}` | Specifies the resources for etcd container in the apiserver | | clustermesh.apiserver.etcd.securityContext | object | `{}` | Security context to be added to clustermesh-apiserver etcd containers | | clustermesh.apiserver.extraArgs | list | `[]` | Additional clustermesh-apiserver arguments. | | clustermesh.apiserver.extraEnv | list | `[]` | Additional clustermesh-apiserver environment variables. | | clustermesh.apiserver.extraVolumeMounts | list | `[]` | Additional clustermesh-apiserver volumeMounts. | | clustermesh.apiserver.extraVolumes | list | `[]` | Additional clustermesh-apiserver volumes. | -| clustermesh.apiserver.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-clustermesh-apiserver","tag":"v1.14.5","useDigest":false}` | Clustermesh API server image. | +| clustermesh.apiserver.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-clustermesh-apiserver","tag":"v1.15.1","useDigest":false}` | Clustermesh API server image. | | clustermesh.apiserver.kvstoremesh.enabled | bool | `false` | Enable KVStoreMesh. KVStoreMesh caches the information retrieved from the remote clusters in the local etcd instance. | | clustermesh.apiserver.kvstoremesh.extraArgs | list | `[]` | Additional KVStoreMesh arguments. | | clustermesh.apiserver.kvstoremesh.extraEnv | list | `[]` | Additional KVStoreMesh environment variables. | | clustermesh.apiserver.kvstoremesh.extraVolumeMounts | list | `[]` | Additional KVStoreMesh volumeMounts. | -| clustermesh.apiserver.kvstoremesh.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"quay.io/cilium/kvstoremesh","tag":"v1.14.5","useDigest":false}` | KVStoreMesh image. | +| clustermesh.apiserver.kvstoremesh.lifecycle | object | `{}` | lifecycle setting for the KVStoreMesh container | | clustermesh.apiserver.kvstoremesh.resources | object | `{}` | Resource requests and limits for the KVStoreMesh container | | clustermesh.apiserver.kvstoremesh.securityContext | object | `{"allowPrivilegeEscalation":false,"capabilities":{"drop":["ALL"]}}` | KVStoreMesh Security context | +| clustermesh.apiserver.lifecycle | object | `{}` | lifecycle setting for the apiserver container | | clustermesh.apiserver.metrics.enabled | bool | `true` | Enables exporting apiserver metrics in OpenMetrics format. | -| clustermesh.apiserver.metrics.etcd.enabled | bool | `false` | Enables exporting etcd metrics in OpenMetrics format. | +| clustermesh.apiserver.metrics.etcd.enabled | bool | `true` | Enables exporting etcd metrics in OpenMetrics format. | | clustermesh.apiserver.metrics.etcd.mode | string | `"basic"` | Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. | | clustermesh.apiserver.metrics.etcd.port | int | `9963` | Configure the port the etcd metric server listens on. | | clustermesh.apiserver.metrics.kvstoremesh.enabled | bool | `true` | Enables exporting KVStoreMesh metrics in OpenMetrics format. | @@ -198,15 +214,13 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.apiserver.service.internalTrafficPolicy | string | `"Cluster"` | The internalTrafficPolicy of service used for apiserver access. | | clustermesh.apiserver.service.nodePort | int | `32379` | Optional port to use as the node port for apiserver access. WARNING: make sure to configure a different NodePort in each cluster if kube-proxy replacement is enabled, as Cilium is currently affected by a known bug (#24692) when NodePorts are handled by the KPR implementation. If a service with the same NodePort exists both in the local and the remote cluster, all traffic originating from inside the cluster and targeting the corresponding NodePort will be redirected to a local backend, regardless of whether the destination node belongs to the local or the remote cluster. | | clustermesh.apiserver.service.type | string | `"NodePort"` | The type of service used for apiserver access. | +| clustermesh.apiserver.terminationGracePeriodSeconds | int | `30` | terminationGracePeriodSeconds for the clustermesh-apiserver deployment | | clustermesh.apiserver.tls.admin | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver admin certificate and private key. Used if 'auto' is not enabled. | | clustermesh.apiserver.tls.authMode | string | `"legacy"` | Configure the clustermesh authentication mode. Supported values: - legacy: All clusters access remote clustermesh instances with the same username (i.e., remote). The "remote" certificate must be generated with CN=remote if provided manually. - migration: Intermediate mode required to upgrade from legacy to cluster (and vice versa) with no disruption. Specifically, it enables the creation of the per-cluster usernames, while still using the common one for authentication. The "remote" certificate must be generated with CN=remote if provided manually (same as legacy). - cluster: Each cluster accesses remote etcd instances with a username depending on the local cluster name (i.e., remote-). The "remote" certificate must be generated with CN=remote- if provided manually. Cluster mode is meaningful only when the same CA is shared across all clusters part of the mesh. | | clustermesh.apiserver.tls.auto | object | `{"certManagerIssuerRef":{},"certValidityDuration":1095,"enabled":true,"method":"helm"}` | Configure automatic TLS certificates generation. A Kubernetes CronJob is used the generate any certificates not provided by the user at installation time. | | clustermesh.apiserver.tls.auto.certManagerIssuerRef | object | `{}` | certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. | | clustermesh.apiserver.tls.auto.certValidityDuration | int | `1095` | Generated certificates validity duration in days. | | clustermesh.apiserver.tls.auto.enabled | bool | `true` | When set to true, automatically generate a CA and certificates to enable mTLS between clustermesh-apiserver and external workload instances. If set to false, the certs to be provided by setting appropriate values below. | -| clustermesh.apiserver.tls.ca | object | `{"cert":"","key":""}` | Deprecated in favor of tls.ca. To be removed in 1.15. base64 encoded PEM values for the ExternalWorkload CA certificate and private key. | -| clustermesh.apiserver.tls.ca.cert | string | `""` | Deprecated in favor of tls.ca.cert. To be removed in 1.15. Optional CA cert. If it is provided, it will be used by the 'cronJob' method to generate all other certificates. Otherwise, an ephemeral CA is generated. | -| clustermesh.apiserver.tls.ca.key | string | `""` | Deprecated in favor of tls.ca.key. To be removed in 1.15. Optional CA private key. If it is provided, it will be used by the 'cronJob' method to generate all other certificates. Otherwise, an ephemeral CA is generated. | | clustermesh.apiserver.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver client certificate and private key. Used if 'auto' is not enabled. | | clustermesh.apiserver.tls.remote | object | `{"cert":"","key":""}` | base64 encoded PEM values for the clustermesh-apiserver remote cluster certificate and private key. Used if 'auto' is not enabled. | | clustermesh.apiserver.tls.server | object | `{"cert":"","extraDnsNames":[],"extraIpAddresses":[],"key":""}` | base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. Used if 'auto' is not enabled. | @@ -219,6 +233,7 @@ contributors across the globe, there is almost always someone available to help. | clustermesh.config.clusters | list | `[]` | List of clusters to be peered in the mesh. | | clustermesh.config.domain | string | `"mesh.cilium.io"` | Default dns domain for the Clustermesh API servers This is used in the case cluster addresses are not provided and IPs are used. | | clustermesh.config.enabled | bool | `false` | Enable the Clustermesh explicit configuration. | +| clustermesh.maxConnectedClusters | int | `255` | The maximum number of clusters to support in a ClusterMesh. This value cannot be changed on running clusters, and all clusters in a ClusterMesh must be configured with the same value. Values > 255 will decrease the maximum allocatable cluster-local identities. Supported values are 255 and 511. | | clustermesh.useAPIServer | bool | `false` | Deploy clustermesh-apiserver for clustermesh | | cni.binPath | string | `"/opt/cni/bin"` | Configure the path to the CNI binary directory on the host. | | cni.chainingMode | string | `"none"` | Configure chaining on top of other CNI plugins. Possible values: - none - aws-cni - flannel - generic-veth - portmap | @@ -231,6 +246,7 @@ contributors across the globe, there is almost always someone available to help. | cni.hostConfDirMountPath | string | `"/host/etc/cni/net.d"` | Configure the path to where the CNI configuration directory is mounted inside the agent pod. | | cni.install | bool | `true` | Install the CNI configuration and binary files into the filesystem. | | cni.logFile | string | `"/var/run/cilium/cilium-cni.log"` | Configure the log file for CNI logging with retention policy of 7 days. Disable CNI file logging by setting this field to empty explicitly. | +| cni.resources | object | `{"requests":{"cpu":"100m","memory":"10Mi"}}` | Specifies the resources for the cni initContainer | | cni.uninstall | bool | `false` | Remove the CNI configuration and binary files on agent shutdown. Enable this if you're removing Cilium from the cluster. Disable this to prevent the CNI configuration file from being removed during agent upgrade, which can cause nodes to go unmanageable. | | conntrackGCInterval | string | `"0s"` | Configure how frequently garbage collection should occur for the datapath connection tracking table. | | conntrackGCMaxInterval | string | `""` | Configure the maximum frequency for the garbage collection of the connection tracking table. Only affects the automatic computation for the frequency and has no effect when 'conntrackGCInterval' is set. This can be set to more frequently clean up unused identities created from ToFQDN policies. | @@ -245,7 +261,7 @@ contributors across the globe, there is almost always someone available to help. | daemon.runPath | string | `"/var/run/cilium"` | Configure where Cilium runtime state should be stored. | | dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":""}` | Grafana dashboards for cilium-agent grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | | debug.enabled | bool | `false` | Enable debug logging | -| debug.verbose | string | `""` | Configure verbosity levels for debug logging This option is used to enable debug messages for operations related to such sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is for enabling debug messages emitted per request, message and connection. Applicable values: - flow - kvstore - envoy - datapath - policy | +| debug.verbose | string | `""` | Configure verbosity levels for debug logging This option is used to enable debug messages for operations related to such sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is for enabling debug messages emitted per request, message and connection. Multiple values can be set via a space-separated string (e.g. "datapath envoy"). Applicable values: - flow - kvstore - envoy - datapath - policy | | defaultPolicies.enabled | bool | `false` | | | defaultPolicies.remove | bool | `false` | | | defaultPolicies.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for default-policies job scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/ | @@ -260,19 +276,18 @@ contributors across the globe, there is almost always someone available to help. | dnsProxy.preCache | string | `""` | DNS cache data at this path is preloaded on agent startup. | | dnsProxy.proxyPort | int | `0` | Global port on which the in-agent DNS proxy should listen. Default 0 is a OS-assigned port. | | dnsProxy.proxyResponseMaxDelay | string | `"100ms"` | The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. | -| egressGateway | object | `{"enabled":false,"installRoutes":false,"reconciliationTriggerInterval":"1s"}` | Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. | -| egressGateway.installRoutes | bool | `false` | Install egress gateway IP rules and routes in order to properly steer egress gateway traffic to the correct ENI interface | +| egressGateway.enabled | bool | `false` | Enables egress gateway to redirect and SNAT the traffic that leaves the cluster. | +| egressGateway.installRoutes | bool | `false` | Deprecated without a replacement necessary. | | egressGateway.reconciliationTriggerInterval | string | `"1s"` | Time between triggers of egress gateway state reconciliations | | eksMode | bool | `false` | | | enableCiliumEndpointSlice | bool | `false` | Enable CiliumEndpointSlice feature. | -| enableCnpStatusUpdates | bool | `false` | Whether to enable CNP status updates. | | enableCriticalPriorityClass | bool | `true` | Explicitly enable or disable priority class. .Capabilities.KubeVersion is unsettable in `helm template` calls, it depends on k8s libraries version that Helm was compiled against. This option allows to explicitly disable setting the priority class, which is useful for rendering charts for gke clusters in advance. | | enableIPv4BIGTCP | bool | `false` | Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods | | enableIPv4Masquerade | bool | `true` | Enables masquerading of IPv4 traffic leaving the node from endpoints. | | enableIPv6BIGTCP | bool | `false` | Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods | | enableIPv6Masquerade | bool | `true` | Enables masquerading of IPv6 traffic leaving the node from endpoints. | -| enableK8sEventHandover | bool | `false` | Configures the use of the KVStore to optimize Kubernetes event handling by mirroring it into the KVstore for reduced overhead in large clusters. | | enableK8sTerminatingEndpoint | bool | `true` | Configure whether to enable auto detect of terminating state for endpoints in order to support graceful termination. | +| enableMasqueradeRouteSource | bool | `false` | Enables masquerading to the source of the route for traffic leaving the node from endpoints. | | enableRuntimeDeviceDetection | bool | `false` | Enables experimental support for the detection of new and removed datapath devices. When devices change the eBPF datapath is reloaded and services updated. If "devices" is set then only those devices, or devices matching a wildcard will be considered. | | enableXTSocketFallback | bool | `true` | Enables the fallback compatibility solution for when the xt_socket kernel module is missing and it is needed for the datapath L7 redirection to work properly. See documentation for details on when this can be disabled: https://docs.cilium.io/en/stable/operations/system_requirements/#linux-kernel. | | encryption.enabled | bool | `false` | Enable transparent network encryption. | @@ -287,7 +302,12 @@ contributors across the globe, there is almost always someone available to help. | encryption.mountPath | string | `"/etc/ipsec"` | Deprecated in favor of encryption.ipsec.mountPath. To be removed in 1.15. Path to mount the secret inside the Cilium pod. This option is only effective when encryption.type is set to ipsec. | | encryption.nodeEncryption | bool | `false` | Enable encryption for pure node to node traffic. This option is only effective when encryption.type is set to "wireguard". | | encryption.secretName | string | `"cilium-ipsec-keys"` | Deprecated in favor of encryption.ipsec.secretName. To be removed in 1.15. Name of the Kubernetes secret containing the encryption keys. This option is only effective when encryption.type is set to ipsec. | +| encryption.strictMode | object | `{"allowRemoteNodeIdentities":false,"cidr":"","enabled":false}` | Configure the WireGuard Pod2Pod strict mode. | +| encryption.strictMode.allowRemoteNodeIdentities | bool | `false` | Allow dynamic lookup of remote node identities. This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. | +| encryption.strictMode.cidr | string | `""` | CIDR for the WireGuard Pod2Pod strict mode. | +| encryption.strictMode.enabled | bool | `false` | Enable WireGuard Pod2Pod strict mode. | | encryption.type | string | `"ipsec"` | Encryption method. Can be either ipsec or wireguard. | +| encryption.wireguard.persistentKeepalive | string | `"0s"` | Controls Wireguard PersistentKeepalive option. Set 0s to disable. | | encryption.wireguard.userspaceFallback | bool | `false` | Enables the fallback to the user-space implementation. | | endpointHealthChecking.enabled | bool | `true` | Enable connectivity health checking between virtual endpoints. | | endpointRoutes.enabled | bool | `false` | Enable use of per endpoint routes instead of routing via the cilium_host interface. | @@ -305,6 +325,7 @@ contributors across the globe, there is almost always someone available to help. | eni.subnetTagsFilter | list | `[]` | Filter via tags (k=v) which will dictate which subnets are going to be used to create new ENIs Important note: This requires that each instance has an ENI with a matching subnet attached when Cilium is deployed. If you only want to control subnets for ENIs attached by Cilium, use the CNI configuration file settings (cni.customConf) instead. | | eni.updateEC2AdapterLimitViaAPI | bool | `true` | Update ENI Adapter limits from the EC2 API | | envoy.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium-envoy"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-envoy. | +| envoy.annotations | object | `{}` | Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) | | envoy.connectTimeoutSeconds | int | `2` | Time in seconds after which a TCP connection attempt times out | | envoy.dnsPolicy | string | `""` | DNS policy for Cilium envoy pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | | envoy.enabled | bool | `false` | Enable Envoy Proxy in standalone DaemonSet. | @@ -316,7 +337,7 @@ contributors across the globe, there is almost always someone available to help. | envoy.extraVolumes | list | `[]` | Additional envoy volumes. | | envoy.healthPort | int | `9878` | TCP port for the health API. | | envoy.idleTimeoutDurationSeconds | int | `60` | Set Envoy upstream HTTP idle connection timeout seconds. Does not apply to connections with pending requests. Default 60s | -| envoy.image | object | `{"digest":"sha256:992998398dadfff7117bfa9fdb7c9474fefab7f0237263f7c8114e106c67baca","override":"","pullPolicy":"IfNotPresent","repository":"quay.io/cilium/cilium-envoy","tag":"v1.26.6-ad82c7c56e88989992fd25d8d67747de865c823b","useDigest":true}` | Envoy container image. | +| envoy.image | object | `{"digest":"sha256:a811830c708296194eaf9cee6c1d22c5f8de3544b7eea6cbdfb810326522a4a2","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-envoy","tag":"v1.27.3-713b673cccf1af661efd75ca20532336517ddcb9","useDigest":true}` | Envoy container image. | | envoy.livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | envoy.livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | | envoy.log.format | string | `"[%Y-%m-%d %T.%e][%t][%l][%n] [%g:%#] %v"` | The format string to use for laying out the log message metadata of Envoy. | @@ -328,14 +349,15 @@ contributors across the globe, there is almost always someone available to help. | envoy.podLabels | object | `{}` | Labels to be added to envoy pods | | envoy.podSecurityContext | object | `{}` | Security Context for cilium-envoy pods. | | envoy.priorityClassName | string | `""` | The priority class to use for cilium-envoy. | +| envoy.prometheus | object | `{"enabled":true,"port":"9964","serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":[],"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Configure Cilium Envoy Prometheus options. Note that some of these apply to either cilium-agent or cilium-envoy. | | envoy.prometheus.enabled | bool | `true` | Enable prometheus metrics for cilium-envoy | | envoy.prometheus.port | string | `"9964"` | Serve prometheus metrics for cilium-envoy on the configured port | | envoy.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-envoy | -| envoy.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | +| envoy.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) Note that this setting applies to both cilium-envoy _and_ cilium-agent with Envoy enabled. | | envoy.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | | envoy.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-envoy | -| envoy.prometheus.serviceMonitor.metricRelabelings | list | `[]` | Metrics relabeling configs for the ServiceMonitor cilium-envoy | -| envoy.prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-envoy | +| envoy.prometheus.serviceMonitor.metricRelabelings | list | `[]` | Metrics relabeling configs for the ServiceMonitor cilium-envoy or for cilium-agent with Envoy configured. | +| envoy.prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-envoy or for cilium-agent with Envoy configured. | | envoy.readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | | envoy.readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | | envoy.resources | object | `{}` | Envoy resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | @@ -352,6 +374,7 @@ contributors across the globe, there is almost always someone available to help. | envoyConfig.secretsNamespace | object | `{"create":true,"name":"cilium-secrets"}` | SecretsNamespace is the namespace in which envoy SDS will retrieve secrets from. | | envoyConfig.secretsNamespace.create | bool | `true` | Create secrets namespace for CiliumEnvoyConfig CRDs. | | envoyConfig.secretsNamespace.name | string | `"cilium-secrets"` | The name of the secret namespace to which Cilium agents are given read access. | +| etcd.annotations | object | `{}` | Annotations to be added to all top-level etcd-operator objects (resources under templates/etcd-operator) | | etcd.clusterDomain | string | `"cluster.local"` | Cluster domain for cilium-etcd-operator. | | etcd.enabled | bool | `false` | Enable etcd mode for the agent. | | etcd.endpoints | list | `["https://CHANGE-ME:2379"]` | List of etcd endpoints (not needed when using managed=true). | @@ -406,24 +429,41 @@ contributors across the globe, there is almost always someone available to help. | hostFirewall | object | `{"enabled":false}` | Configure the host firewall. | | hostFirewall.enabled | bool | `false` | Enables the enforcement of host policies in the eBPF datapath. | | hostPort.enabled | bool | `false` | Enable hostPort service support. | +| hubble.annotations | object | `{}` | Annotations to be added to all top-level hubble objects (resources under templates/hubble) | | hubble.enabled | bool | `true` | Enable Hubble (true by default). | +| hubble.export | object | `{"dynamic":{"config":{"configMapName":"cilium-flowlog-config","content":[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}],"createConfigMap":true},"enabled":false},"fileMaxBackups":5,"fileMaxSizeMb":10,"static":{"allowList":[],"denyList":[],"enabled":false,"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log"}}` | Hubble flows export. | +| hubble.export.dynamic | object | `{"config":{"configMapName":"cilium-flowlog-config","content":[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}],"createConfigMap":true},"enabled":false}` | - Dynamic exporters configuration. Dynamic exporters may be reconfigured without a need of agent restarts. | +| hubble.export.dynamic.config.configMapName | string | `"cilium-flowlog-config"` | -- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. | +| hubble.export.dynamic.config.content | list | `[{"excludeFilters":[],"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log","includeFilters":[],"name":"all"}]` | -- Exporters configuration in YAML format. | +| hubble.export.dynamic.config.createConfigMap | bool | `true` | -- True if helm installer should create config map. Switch to false if you want to self maintain the file content. | +| hubble.export.fileMaxBackups | int | `5` | - Defines max number of backup/rotated files. | +| hubble.export.fileMaxSizeMb | int | `10` | - Defines max file size of output file before it gets rotated. | +| hubble.export.static | object | `{"allowList":[],"denyList":[],"enabled":false,"fieldMask":[],"filePath":"/var/run/cilium/hubble/events.log"}` | - Static exporter configuration. Static exporter is bound to agent lifecycle. | | hubble.listenAddress | string | `":4244"` | An additional address for Hubble to listen to. Set this field ":4244" if you are enabling Hubble Relay, as it assumes that Hubble is listening on port 4244. | -| hubble.metrics | object | `{"dashboards":{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":""},"enableOpenMetrics":false,"enabled":"","port":9965,"serviceAnnotations":{},"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":[],"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Hubble metrics configuration. See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics for more comprehensive documentation about Hubble metrics. | +| hubble.metrics | object | `{"dashboards":{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":""},"enableOpenMetrics":false,"enabled":"","port":9965,"serviceAnnotations":{},"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":[],"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]}}` | Hubble metrics configuration. See https://docs.cilium.io/en/stable/observability/metrics/#hubble-metrics for more comprehensive documentation about Hubble metrics. | | hubble.metrics.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":""}` | Grafana dashboards for hubble grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | | hubble.metrics.enableOpenMetrics | bool | `false` | Enables exporting hubble metrics in OpenMetrics format. | -| hubble.metrics.enabled | string | `""` | Configures the list of metrics to collect. If empty or null, metrics are disabled. Example: enabled: - dns:query;ignoreAAAA - drop - tcp - flow - icmp - http You can specify the list of metrics from the helm CLI: --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" | +| hubble.metrics.enabled | string | `""` | Configures the list of metrics to collect. If empty or null, metrics are disabled. Example: enabled: - dns:query;ignoreAAAA - drop - tcp - flow - icmp - http You can specify the list of metrics from the helm CLI: --set hubble.metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" | | hubble.metrics.port | int | `9965` | Configure the port the hubble metric server listens on. | | hubble.metrics.serviceAnnotations | object | `{}` | Annotations to be added to hubble-metrics service. | | hubble.metrics.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor hubble | | hubble.metrics.serviceMonitor.enabled | bool | `false` | Create ServiceMonitor resources for Prometheus Operator. This requires the prometheus CRDs to be available. ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | | hubble.metrics.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| hubble.metrics.serviceMonitor.jobLabel | string | `""` | jobLabel to add for ServiceMonitor hubble | | hubble.metrics.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor hubble | | hubble.metrics.serviceMonitor.metricRelabelings | list | `[]` | Metrics relabeling configs for the ServiceMonitor hubble | | hubble.metrics.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor hubble | | hubble.peerService.clusterDomain | string | `"cluster.local"` | The cluster domain to use to query the Hubble Peer service. It should be the local cluster. | | hubble.peerService.targetPort | int | `4244` | Target Port for the Peer service, must match the hubble.listenAddress' port. | | hubble.preferIpv6 | bool | `false` | Whether Hubble should prefer to announce IPv6 or IPv4 addresses if both are available. | +| hubble.redact | object | `{"enabled":false,"http":{"headers":{"allow":[],"deny":[]},"urlQuery":false,"userInfo":true},"kafka":{"apiKey":false}}` | Enables redacting sensitive information present in Layer 7 flows. | +| hubble.redact.http.headers.allow | list | `[]` | List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. Example: redact: enabled: true http: headers: allow: - traceparent - tracestate - Cache-Control You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" | +| hubble.redact.http.headers.deny | list | `[]` | List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. Example: redact: enabled: true http: headers: deny: - Authorization - Proxy-Authorization You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" | +| hubble.redact.http.urlQuery | bool | `false` | Enables redacting URL query (GET) parameters. Example: redact: enabled: true http: urlQuery: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.urlQuery="true" | +| hubble.redact.http.userInfo | bool | `true` | Enables redacting user info, e.g., password when basic auth is used. Example: redact: enabled: true http: userInfo: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.http.userInfo="true" | +| hubble.redact.kafka.apiKey | bool | `false` | Enables redacting Kafka's API key. Example: redact: enabled: true kafka: apiKey: true You can specify the options from the helm CLI: --set hubble.redact.enabled="true" --set hubble.redact.kafka.apiKey="true" | | hubble.relay.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for hubble-replay | +| hubble.relay.annotations | object | `{}` | Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) | | hubble.relay.dialTimeout | string | `""` | Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). | | hubble.relay.enabled | bool | `true` | Enable Hubble Relay (requires hubble.enabled=true) | | hubble.relay.extraEnv | list | `[]` | Additional hubble-relay environment variables. | @@ -431,7 +471,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.extraVolumes | list | `[]` | Additional hubble-relay volumes. | | hubble.relay.gops.enabled | bool | `true` | Enable gops for hubble-relay | | hubble.relay.gops.port | int | `9893` | Configure gops listen port for hubble-relay | -| hubble.relay.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/hubble-relay","tag":"v1.14.5","useDigest":false}` | Hubble-relay container image. | +| hubble.relay.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/hubble-relay","tag":"v1.15.1","useDigest":false}` | Hubble-relay container image. | | hubble.relay.listenHost | string | `""` | Host to listen to. Specify an empty string to bind to all the interfaces. | | hubble.relay.listenPort | string | `"4245"` | Port to listen to. | | hubble.relay.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | @@ -462,9 +502,9 @@ contributors across the globe, there is almost always someone available to help. | hubble.relay.sortBufferDrainTimeout | string | `""` | When the per-request flows sort buffer is not full, a flow is drained every time this timeout is reached (only affects requests in follow-mode) (e.g. "1s"). | | hubble.relay.sortBufferLenMax | int | `100` | Max number of flows that can be buffered for sorting before being sent to the client (per request) (e.g. 100). | | hubble.relay.terminationGracePeriodSeconds | int | `1` | Configure termination grace period for hubble relay Deployment. | -| hubble.relay.tls | object | `{"client":{"cert":"","key":""},"server":{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false}}` | TLS configuration for Hubble Relay | +| hubble.relay.tls | object | `{"client":{"cert":"","key":""},"server":{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false,"relayName":"ui.hubble-relay.cilium.io"}}` | TLS configuration for Hubble Relay | | hubble.relay.tls.client | object | `{"cert":"","key":""}` | base64 encoded PEM values for the hubble-relay client certificate and private key This keypair is presented to Hubble server instances for mTLS authentication and is required when hubble.tls.enabled is true. These values need to be set manually if hubble.tls.auto.enabled is false. | -| hubble.relay.tls.server | object | `{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false}` | base64 encoded PEM values for the hubble-relay server certificate and private key | +| hubble.relay.tls.server | object | `{"cert":"","enabled":false,"extraDnsNames":[],"extraIpAddresses":[],"key":"","mtls":false,"relayName":"ui.hubble-relay.cilium.io"}` | base64 encoded PEM values for the hubble-relay server certificate and private key | | hubble.relay.tls.server.extraDnsNames | list | `[]` | extra DNS names added to certificate when its auto gen | | hubble.relay.tls.server.extraIpAddresses | list | `[]` | extra IP addresses added to certificate when its auto gen | | hubble.relay.tolerations | list | `[]` | Node tolerations for pod assignment on nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | @@ -484,10 +524,13 @@ contributors across the globe, there is almost always someone available to help. | hubble.tls.server.extraDnsNames | list | `[]` | Extra DNS names added to certificate when it's auto generated | | hubble.tls.server.extraIpAddresses | list | `[]` | Extra IP addresses added to certificate when it's auto generated | | hubble.ui.affinity | object | `{}` | Affinity for hubble-ui | +| hubble.ui.annotations | object | `{}` | Annotations to be added to all top-level hubble-ui objects (resources under templates/hubble-ui) | | hubble.ui.backend.extraEnv | list | `[]` | Additional hubble-ui backend environment variables. | | hubble.ui.backend.extraVolumeMounts | list | `[]` | Additional hubble-ui backend volumeMounts. | | hubble.ui.backend.extraVolumes | list | `[]` | Additional hubble-ui backend volumes. | -| hubble.ui.backend.image | object | `{"digest":"sha256:1f86f3400827a0451e6332262467f894eeb7caf0eb8779bd951e2caa9d027cbe","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/hubble-ui-backend","tag":"v0.12.1","useDigest":false}` | Hubble-ui backend image. | +| hubble.ui.backend.image | object | `{"digest":"sha256:1e7657d997c5a48253bb8dc91ecee75b63018d16ff5e5797e5af367336bc8803","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/hubble-ui-backend","tag":"v0.13.0","useDigest":false}` | Hubble-ui backend image. | +| hubble.ui.backend.livenessProbe.enabled | bool | `false` | Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) | +| hubble.ui.backend.readinessProbe.enabled | bool | `false` | Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) | | hubble.ui.backend.resources | object | `{}` | Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. | | hubble.ui.backend.securityContext | object | `{}` | Hubble-ui backend security context. | | hubble.ui.baseUrl | string | `"/"` | Defines base url prefix for all hubble-ui http requests. It needs to be changed in case if ingress for hubble-ui is configured under some sub-path. Trailing `/` is required for custom path, ex. `/service-map/` | @@ -495,7 +538,7 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.frontend.extraEnv | list | `[]` | Additional hubble-ui frontend environment variables. | | hubble.ui.frontend.extraVolumeMounts | list | `[]` | Additional hubble-ui frontend volumeMounts. | | hubble.ui.frontend.extraVolumes | list | `[]` | Additional hubble-ui frontend volumes. | -| hubble.ui.frontend.image | object | `{"digest":"sha256:9e5f81ee747866480ea1ac4630eb6975ff9227f9782b7c93919c081c33f38267","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/hubble-ui","tag":"v0.12.1","useDigest":false}` | Hubble-ui frontend image. | +| hubble.ui.frontend.image | object | `{"digest":"sha256:7d663dc16538dd6e29061abd1047013a645e6e69c115e008bee9ea9fef9a6666","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/hubble-ui","tag":"v0.13.0","useDigest":false}` | Hubble-ui frontend image. | | hubble.ui.frontend.resources | object | `{}` | Resource requests and limits for the 'frontend' container of the 'hubble-ui' deployment. | | hubble.ui.frontend.securityContext | object | `{}` | Hubble-ui frontend security context. | | hubble.ui.frontend.server.ipv6 | object | `{"enabled":true}` | Controls server listener for ipv6 | @@ -521,14 +564,15 @@ contributors across the globe, there is almost always someone available to help. | hubble.ui.updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":1},"type":"RollingUpdate"}` | hubble-ui update strategy. | | identityAllocationMode | string | `"crd"` | Method to use for identity allocation (`crd` or `kvstore`). | | identityChangeGracePeriod | string | `"5s"` | Time to wait before using new identity on endpoint identity change. | -| image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","registry":"gsoci.azurecr.io","repository":"giantswarm/cilium","tag":"v1.14.5","useDigest":false}` | Agent container image. | +| image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","registry":"gsoci.azurecr.io","repository":"giantswarm/cilium","tag":"v1.15.1","useDigest":false}` | Agent container image. | | imagePullSecrets | list | `[]` | Configure image pull secrets for pulling container images | | ingressController.default | bool | `false` | Set cilium ingress controller to be the default ingress controller This will let cilium ingress controller route entries without ingress class set | | ingressController.defaultSecretName | string | `""` | Default secret name for ingresses without .spec.tls[].secretName set. | | ingressController.defaultSecretNamespace | string | `""` | Default secret namespace for ingresses without .spec.tls[].secretName set. | +| ingressController.enableProxyProtocol | bool | `false` | Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. | | ingressController.enabled | bool | `false` | Enable cilium ingress controller This will automatically set enable-envoy-config as well. | | ingressController.enforceHttps | bool | `true` | Enforce https for host having matching TLS host in Ingress. Incoming traffic to http listener will return 308 http error code with respective location in header. | -| ingressController.ingressLBAnnotationPrefixes | list | `["service.beta.kubernetes.io","service.kubernetes.io","cloud.google.com"]` | IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate from Ingress to the Load Balancer service | +| ingressController.ingressLBAnnotationPrefixes | list | `["service.beta.kubernetes.io","service.kubernetes.io","cloud.google.com"]` | IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service | | ingressController.loadbalancerMode | string | `"dedicated"` | Default ingress load balancer mode Supported values: shared, dedicated For granular control, use the following annotations on the ingress resource ingress.cilium.io/loadbalancer-mode: shared|dedicated, | | ingressController.secretsNamespace | object | `{"create":true,"name":"cilium-secrets","sync":true}` | SecretsNamespace is the namespace in which envoy SDS will retrieve TLS secrets from. | | ingressController.secretsNamespace.create | bool | `true` | Create secrets namespace for Ingress. | @@ -562,8 +606,8 @@ contributors across the globe, there is almost always someone available to help. | ipv6NativeRoutingCIDR | string | `""` | Allows to explicitly specify the IPv6 CIDR for native routing. When specified, Cilium assumes networking for this CIDR is preconfigured and hands traffic destined for that range to the Linux network stack without applying any SNAT. Generally speaking, specifying a native routing CIDR implies that Cilium can depend on the underlying networking stack to route packets to their destination. To offer a concrete example, if Cilium is configured to use direct routing and the Kubernetes CIDR is included in the native routing CIDR, the user must configure the routes to reach pods, either manually or by setting the auto-direct-node-routes flag. | | k8s | object | `{}` | Configure Kubernetes specific configuration | | k8sClientRateLimit | object | `{"burst":10,"qps":5}` | Configure the client side rate limit for the agent and operator If the amount of requests to the Kubernetes API server exceeds the configured rate limit, the agent and operator will start to throttle requests by delaying them until there is budget or the request times out. | -| k8sClientRateLimit.burst | int | `10` | The burst request rate in requests per second. The rate limiter will allow short bursts with a higher rate. | -| k8sClientRateLimit.qps | int | `5` | The sustained request rate in requests per second. | +| k8sClientRateLimit.burst | int | 10 for k8s up to 1.26. 20 for k8s version 1.27+ | The burst request rate in requests per second. The rate limiter will allow short bursts with a higher rate. | +| k8sClientRateLimit.qps | int | 5 for k8s up to 1.26. 10 for k8s version 1.27+ | The sustained request rate in requests per second. | | k8sNetworkPolicy.enabled | bool | `true` | Enable support for K8s NetworkPolicy | | k8sServiceHost | string | `""` | Kubernetes service host | | k8sServicePort | string | `""` | Kubernetes service port | @@ -582,7 +626,8 @@ contributors across the globe, there is almost always someone available to help. | labels | string | `"k8s:!.*/enforce k8s:!.*fluxcd.io/.* k8s:!.*kubernetes.io/managed-by.* k8s:!controller-uid k8s:!job-name"` | | | livenessProbe.failureThreshold | int | `10` | failure threshold of liveness probe | | livenessProbe.periodSeconds | int | `30` | interval between checks of the liveness probe | -| loadBalancer | object | `{"l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | +| loadBalancer | object | `{"acceleration":"disabled","l7":{"algorithm":"round_robin","backend":"disabled","ports":[]}}` | Configure service load balancing | +| loadBalancer.acceleration | string | `"disabled"` | acceleration is the option to accelerate service handling via XDP Applicable values can be: disabled (do not use XDP), native (XDP BPF program is run directly out of the networking driver's early receive path), or best-effort (use native mode XDP acceleration on devices that support it). | | loadBalancer.l7 | object | `{"algorithm":"round_robin","backend":"disabled","ports":[]}` | L7 LoadBalancer | | loadBalancer.l7.algorithm | string | `"round_robin"` | Default LB algorithm The default LB algorithm to be used for services, which can be overridden by the service annotation (e.g. service.cilium.io/lb-l7-algorithm) Applicable values: round_robin, least_request, random | | loadBalancer.l7.backend | string | `"disabled"` | Enable L7 service load balancing via envoy proxy. The request to a k8s service, which has specific annotation e.g. service.cilium.io/lb-l7, will be forwarded to the local backend proxy to be load balanced to the service endpoints. Please refer to docs for supported annotations for more configuration. Applicable values: - envoy: Enable L7 load balancing via envoy proxy. This will automatically set enable-envoy-config as well. - disabled: Disable L7 load balancing by way of service annotation. | @@ -595,13 +640,15 @@ contributors across the globe, there is almost always someone available to help. | name | string | `"cilium"` | Agent container name. | | nat46x64Gateway | object | `{"enabled":false}` | Configure standalone NAT46/NAT64 gateway | | nat46x64Gateway.enabled | bool | `false` | Enable RFC8215-prefixed translation | -| nodePort | object | `{"autoProtectPortRange":true,"bindProtection":true,"enableHealthCheck":true,"enabled":false}` | Configure N-S k8s service loadbalancing | +| nodePort | object | `{"autoProtectPortRange":true,"bindProtection":true,"enableHealthCheck":true,"enableHealthCheckLoadBalancerIP":false,"enabled":false}` | Configure N-S k8s service loadbalancing | | nodePort.autoProtectPortRange | bool | `true` | Append NodePort range to ip_local_reserved_ports if clash with ephemeral ports is detected. | | nodePort.bindProtection | bool | `true` | Set to true to prevent applications binding to service ports. | | nodePort.enableHealthCheck | bool | `true` | Enable healthcheck nodePort server for NodePort services | +| nodePort.enableHealthCheckLoadBalancerIP | bool | `false` | Enable access of the healthcheck nodePort on the LoadBalancerIP. Needs EnableHealthCheck to be enabled | | nodePort.enabled | bool | `false` | Enable the Cilium NodePort service implementation. | | nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node selector for cilium-agent. | | nodeinit.affinity | object | `{}` | Affinity for cilium-nodeinit | +| nodeinit.annotations | object | `{}` | Annotations to be added to all top-level nodeinit objects (resources under templates/cilium-nodeinit) | | nodeinit.bootstrapFile | string | `"/tmp/cilium-bootstrap.d/cilium-bootstrap-time"` | bootstrapFile is the location of the file where the bootstrap timestamp is written by the node-init DaemonSet | | nodeinit.enabled | bool | `false` | Enable the node initialization DaemonSet | | nodeinit.extraEnv | list | `[]` | Additional nodeinit environment variables. | @@ -619,6 +666,7 @@ contributors across the globe, there is almost always someone available to help. | nodeinit.tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for nodeinit scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | | nodeinit.updateStrategy | object | `{"type":"RollingUpdate"}` | node-init update strategy | | operator.affinity | object | `{"podAntiAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"io.cilium/app":"operator"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-operator | +| operator.annotations | object | `{}` | Annotations to be added to all top-level cilium-operator objects (resources under templates/cilium-operator) | | operator.dashboards | object | `{"annotations":{},"enabled":false,"label":"grafana_dashboard","labelValue":"1","namespace":""}` | Grafana dashboards for cilium-operator grafana can import dashboards based on the label and value ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards | | operator.dnsPolicy | string | `""` | DNS policy for Cilium operator pods. Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy | | operator.enabled | bool | `true` | Enable the cilium-operator component (required). | @@ -630,7 +678,7 @@ contributors across the globe, there is almost always someone available to help. | operator.extraVolumes | list | `[]` | Additional cilium-operator volumes. | | operator.identityGCInterval | string | `"15m0s"` | Interval for identity garbage collection. | | operator.identityHeartbeatTimeout | string | `"30m0s"` | Timeout for identity heartbeats. | -| operator.image | object | `{"alibabacloudDigest":"","awsDigest":"","azureDigest":"","genericDigest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-operator","suffix":"","tag":"v1.14.5","useDigest":false}` | cilium-operator image. | +| operator.image | object | `{"alibabacloudDigest":"","awsDigest":"","azureDigest":"","genericDigest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium-operator","suffix":"","tag":"v1.15.1","useDigest":false}` | cilium-operator image. | | operator.nodeGCInterval | string | `"5m0s"` | Interval for cilium node garbage collection. | | operator.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for cilium-operator pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | operator.podAnnotations | object | `{}` | Annotations to be added to cilium-operator pods | @@ -642,10 +690,11 @@ contributors across the globe, there is almost always someone available to help. | operator.pprof.enabled | bool | `false` | Enable pprof for cilium-operator | | operator.pprof.port | int | `6061` | Configure pprof listen port for cilium-operator | | operator.priorityClassName | string | `""` | The priority class to use for cilium-operator | -| operator.prometheus | object | `{"enabled":true,"port":9963,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":[],"relabelings":[]}}` | Enable prometheus metrics for cilium-operator on the configured port at /metrics | +| operator.prometheus | object | `{"enabled":true,"port":9963,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":[],"relabelings":[]}}` | Enable prometheus metrics for cilium-operator on the configured port at /metrics | | operator.prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-operator | | operator.prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | | operator.prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| operator.prometheus.serviceMonitor.jobLabel | string | `""` | jobLabel to add for ServiceMonitor cilium-operator | | operator.prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-operator | | operator.prometheus.serviceMonitor.metricRelabelings | list | `[]` | Metrics relabeling configs for the ServiceMonitor cilium-operator | | operator.prometheus.serviceMonitor.relabelings | list | `[]` | Relabeling configs for the ServiceMonitor cilium-operator | @@ -667,16 +716,18 @@ contributors across the globe, there is almost always someone available to help. | podAnnotations | object | `{}` | Annotations to be added to agent pods | | podLabels | object | `{}` | Labels to be added to agent pods | | podSecurityContext | object | `{}` | Security Context for cilium-agent pods. | +| policyCIDRMatchMode | list | `[]` | policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. The possible value is "nodes". | | policyEnforcementMode | string | `"default"` | The agent can be put into one of the three policy enforcement modes: default, always and never. ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes | | pprof.address | string | `"localhost"` | Configure pprof listen address for cilium-agent | | pprof.enabled | bool | `false` | Enable pprof for cilium-agent | | pprof.port | int | `6060` | Configure pprof listen port for cilium-agent | | preflight.affinity | object | `{"podAffinity":{"requiredDuringSchedulingIgnoredDuringExecution":[{"labelSelector":{"matchLabels":{"k8s-app":"cilium"}},"topologyKey":"kubernetes.io/hostname"}]}}` | Affinity for cilium-preflight | +| preflight.annotations | object | `{}` | Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) | | preflight.enabled | bool | `false` | Enable Cilium pre-flight resources (required for upgrade) | | preflight.extraEnv | list | `[]` | Additional preflight environment variables. | | preflight.extraVolumeMounts | list | `[]` | Additional preflight volumeMounts. | | preflight.extraVolumes | list | `[]` | Additional preflight volumes. | -| preflight.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium","tag":"v1.14.5","useDigest":false}` | Cilium pre-flight image. | +| preflight.image | object | `{"digest":"","override":"","pullPolicy":"IfNotPresent","repository":"giantswarm/cilium","tag":"v1.15.1","useDigest":false}` | Cilium pre-flight image. | | preflight.nodeSelector | object | `{"kubernetes.io/os":"linux"}` | Node labels for preflight pod assignment ref: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#nodeselector | | preflight.podAnnotations | object | `{}` | Annotations to be added to preflight pods | | preflight.podDisruptionBudget.enabled | bool | `false` | enable PodDisruptionBudget ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ | @@ -692,11 +743,13 @@ contributors across the globe, there is almost always someone available to help. | preflight.updateStrategy | object | `{"type":"RollingUpdate"}` | preflight update strategy | | preflight.validateCNPs | bool | `true` | By default we should always validate the installed CNPs before upgrading Cilium. This will make sure the user will have the policies deployed in the cluster with the right schema. | | priorityClassName | string | `""` | The priority class to use for cilium-agent. | -| prometheus | object | `{"enabled":true,"metrics":["+cilium_bpf_map_pressure"],"port":9962,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","labels":{},"metricRelabelings":[],"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}],"trustCRDsExist":false}}` | Configure prometheus metrics on the configured port at /metrics | +| prometheus | object | `{"controllerGroupMetrics":["write-cni-file","sync-host-ips","sync-lb-maps-with-k8s-services"],"enabled":true,"metrics":["+cilium_bpf_map_pressure"],"port":9962,"serviceMonitor":{"annotations":{},"enabled":false,"interval":"10s","jobLabel":"","labels":{},"metricRelabelings":[],"relabelings":[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}],"trustCRDsExist":false}}` | Configure prometheus metrics on the configured port at /metrics | +| prometheus.controllerGroupMetrics | list | `["write-cni-file","sync-host-ips","sync-lb-maps-with-k8s-services"]` | - Enable controller group metrics for monitoring specific Cilium subsystems. The list is a list of controller group names. The special values of "all" and "none" are supported. The set of controller group names is not guaranteed to be stable between Cilium versions. | | prometheus.metrics | list | `["+cilium_bpf_map_pressure"]` | Metrics that should be enabled or disabled from the default metric list. The list is expected to be separated by a space. (+metric_foo to enable metric_foo , -metric_bar to disable metric_bar). ref: https://docs.cilium.io/en/stable/observability/metrics/ | | prometheus.serviceMonitor.annotations | object | `{}` | Annotations to add to ServiceMonitor cilium-agent | | prometheus.serviceMonitor.enabled | bool | `false` | Enable service monitors. This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) | | prometheus.serviceMonitor.interval | string | `"10s"` | Interval for scrape metrics. | +| prometheus.serviceMonitor.jobLabel | string | `""` | jobLabel to add for ServiceMonitor cilium-agent | | prometheus.serviceMonitor.labels | object | `{}` | Labels to add to ServiceMonitor cilium-agent | | prometheus.serviceMonitor.metricRelabelings | list | `[]` | Metrics relabeling configs for the ServiceMonitor cilium-agent | | prometheus.serviceMonitor.relabelings | list | `[{"replacement":"${1}","sourceLabels":["__meta_kubernetes_pod_node_name"],"targetLabel":"node"}]` | Relabeling configs for the ServiceMonitor cilium-agent | @@ -708,7 +761,7 @@ contributors across the globe, there is almost always someone available to help. | rbac.create | bool | `true` | Enable creation of Resource-Based Access Control configuration. | | readinessProbe.failureThreshold | int | `3` | failure threshold of readiness probe | | readinessProbe.periodSeconds | int | `30` | interval between checks of the readiness probe | -| remoteNodeIdentity | bool | `true` | Enable use of the remote node identity. ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity | +| remoteNodeIdentity | bool | `true` | Enable use of the remote node identity. ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity Deprecated without replacement in 1.15. To be removed in 1.16. | | resourceQuotas | object | `{"cilium":{"hard":{"pods":"10k"}},"enabled":false,"operator":{"hard":{"pods":"15"}}}` | Enable resource quotas for priority classes used in the cluster. | | resources | object | `{}` | Agent resource limits & requests ref: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ | | rollOutCiliumPods | bool | `false` | Roll out cilium agent pods automatically when configmap is updated. | @@ -725,6 +778,7 @@ contributors across the globe, there is almost always someone available to help. | serviceAccounts.clustermeshcertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"clustermesh-apiserver-generate-certs"}` | Clustermeshcertgen is used if clustermesh.apiserver.tls.auto.method=cronJob | | serviceAccounts.hubblecertgen | object | `{"annotations":{},"automount":true,"create":true,"name":"hubble-generate-certs"}` | Hubblecertgen is used if hubble.tls.auto.method=cronJob | | serviceAccounts.nodeinit.enabled | bool | `false` | Enabled is temporary until https://github.com/cilium/cilium-cli/issues/1396 is implemented. Cilium CLI doesn't create the SAs for node-init, thus the workaround. Helm is not affected by this issue. Name and automount can be configured, if enabled is set to true. Otherwise, they are ignored. Enabled can be removed once the issue is fixed. Cilium-nodeinit DS must also be fixed. | +| serviceNoBackendResponse | string | `"reject"` | Configure what the response should be to traffic for a service without backends. "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". Possible values: - reject (default) - drop | | sleepAfterInit | bool | `false` | Do not run Cilium agent when running with clean mode. Useful to completely uninstall Cilium as it will stop Cilium from starting and create artifacts in the node. | | socketLB | object | `{"enabled":false}` | Configure socket LB | | socketLB.enabled | bool | `false` | Enable socket LB | @@ -745,7 +799,6 @@ contributors across the globe, there is almost always someone available to help. | tls.caBundle.useSecret | bool | `false` | Use a Secret instead of a ConfigMap. | | tls.secretsBackend | string | `"local"` | This configures how the Cilium agent loads the secrets used TLS-aware CiliumNetworkPolicies (namely the secrets referenced by terminatingTLS and originatingTLS). Possible values: - local - k8s | | tolerations | list | `[{"operator":"Exists"}]` | Node tolerations for agent scheduling to nodes with taints ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ | -| tunnel | string | `"vxlan"` | Configure the encapsulation configuration for communication between nodes. Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. Possible values: - disabled - vxlan - geneve | | tunnelPort | int | Port 8472 for VXLAN, Port 6081 for Geneve | Configure VXLAN and Geneve tunnel port. | | tunnelProtocol | string | `"vxlan"` | Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. Possible values: - "" - vxlan - geneve | | updateStrategy | object | `{"rollingUpdate":{"maxUnavailable":2},"type":"RollingUpdate"}` | Cilium agent update strategy | diff --git a/helm/cilium/files/agent/poststart-eni.bash b/helm/cilium/files/agent/poststart-eni.bash index 66fccf45..a57d8968 100644 --- a/helm/cilium/files/agent/poststart-eni.bash +++ b/helm/cilium/files/agent/poststart-eni.bash @@ -11,9 +11,9 @@ set -o nounset # dependencies on anything that is part of the startup script # itself, and can be safely run multiple times per node (e.g. in # case of a restart). -if [[ "$(iptables-save | grep -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; +if [[ "$(iptables-save | grep -E -c 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN')" != "0" ]]; then echo 'Deleting iptables rules created by the AWS CNI VPC plugin' - iptables-save | grep -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore + iptables-save | grep -E -v 'AWS-SNAT-CHAIN|AWS-CONNMARK-CHAIN' | iptables-restore fi echo 'Done!' diff --git a/helm/cilium/files/cilium-agent/dashboards/cilium-dashboard.json b/helm/cilium/files/cilium-agent/dashboards/cilium-dashboard.json index 997608ee..94af2eac 100644 --- a/helm/cilium/files/cilium-agent/dashboards/cilium-dashboard.json +++ b/helm/cilium/files/cilium-agent/dashboards/cilium-dashboard.json @@ -27,7 +27,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -131,7 +134,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -271,7 +277,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -394,7 +403,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -511,7 +523,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -636,7 +651,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "BPF memory usage in the entire system including components not managed by Cilium.", "fieldConfig": { "defaults": { @@ -759,7 +777,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "description": "Fill percentage of BPF maps, tagged by map name", "fieldConfig": { "defaults": { @@ -870,7 +891,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -971,7 +995,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1072,7 +1099,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1173,7 +1203,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1274,7 +1307,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1375,7 +1411,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1511,7 +1550,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1612,7 +1654,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "decimals": 2, "fieldConfig": { "defaults": { @@ -1715,7 +1760,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1816,7 +1864,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -1915,7 +1966,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2016,7 +2070,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2117,7 +2174,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2239,7 +2299,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "decimals": 2, "fieldConfig": { "defaults": { @@ -2342,7 +2405,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "decimals": 2, "fieldConfig": { "defaults": { @@ -2445,7 +2511,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2546,7 +2615,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2647,7 +2719,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2767,7 +2842,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2864,7 +2942,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -2984,7 +3065,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3150,7 +3234,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3316,7 +3403,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3482,7 +3572,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3633,7 +3726,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "decimals": null, "fieldConfig": { "defaults": { @@ -3740,7 +3836,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3837,7 +3936,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -3934,7 +4036,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4047,7 +4152,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4147,7 +4255,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4270,7 +4381,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4370,7 +4484,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4518,7 +4635,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4638,7 +4758,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4740,7 +4863,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4864,7 +4990,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -4966,7 +5095,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5102,7 +5234,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5218,7 +5353,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5327,7 +5465,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5455,7 +5596,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5591,7 +5735,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5676,7 +5823,7 @@ "refId": "C" }, { - "expr": "sum(cilium_policy_change_total{k8s_app=\"cilium\", pod=~\"$pod\"}, outcome=\"fail\") by (pod)", + "expr": "sum(cilium_policy_change_total{k8s_app=\"cilium\", pod=~\"$pod\", outcome=\"fail\"}) by (pod)", "format": "time_series", "intervalFactor": 1, "legendFormat": "policy change errors", @@ -5733,7 +5880,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5841,7 +5991,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -5983,7 +6136,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "decimals": null, "fieldConfig": { "defaults": { @@ -6083,7 +6239,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "decimals": null, "fieldConfig": { "defaults": { @@ -6188,7 +6347,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6298,7 +6460,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6421,7 +6586,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6542,7 +6710,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6674,7 +6845,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6775,7 +6949,10 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6876,7 +7053,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -6977,7 +7157,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7078,7 +7261,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7178,7 +7364,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7277,7 +7466,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7376,7 +7568,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7475,7 +7670,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7578,7 +7776,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7681,7 +7882,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7784,7 +7988,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7883,7 +8090,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -7982,7 +8192,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -8081,7 +8294,10 @@ "bars": true, "dashLength": 10, "dashes": false, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "custom": {} @@ -8182,6 +8398,21 @@ "tags": [], "templating": { "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, { "allValue": "cilium.*", "current": { @@ -8189,7 +8420,10 @@ "text": "All", "value": "$__all" }, - "datasource": "prometheus", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "definition": "label_values(cilium_version, pod)", "hide": 0, "includeAll": true, diff --git a/helm/cilium/files/cilium-envoy/configmap/bootstrap-config.json b/helm/cilium/files/cilium-envoy/configmap/bootstrap-config.json index ea8984db..87939f69 100644 --- a/helm/cilium/files/cilium-envoy/configmap/bootstrap-config.json +++ b/helm/cilium/files/cilium-envoy/configmap/bootstrap-config.json @@ -301,6 +301,14 @@ "resourceApiVersion": "V3" } }, + "bootstrapExtensions": [ + { + "name": "envoy.bootstrap.internal_listener", + "typed_config": { + "@type": "type.googleapis.com/envoy.extensions.bootstrap.internal_listener.v3.InternalListener" + } + } + ], "layeredRuntime": { "layers": [ { diff --git a/helm/cilium/files/hubble/dashboards/hubble-dashboard.json b/helm/cilium/files/hubble/dashboards/hubble-dashboard.json index 12de657d..8de5ec1d 100644 --- a/helm/cilium/files/hubble/dashboards/hubble-dashboard.json +++ b/helm/cilium/files/hubble/dashboards/hubble-dashboard.json @@ -3226,7 +3226,7 @@ ] }, "timezone": "", - "title": "Hubble", + "title": "Hubble Metrics and Monitoring", "uid": "5HftnJAWz", "version": 24 } diff --git a/helm/cilium/files/hubble/dashboards/hubble-dns-namespace.json b/helm/cilium/files/hubble/dashboards/hubble-dns-namespace.json new file mode 100644 index 00000000..d286fdb3 --- /dev/null +++ b/helm/cilium/files/hubble/dashboards/hubble-dns-namespace.json @@ -0,0 +1,602 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.7" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 16612, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cilium-overview" + ], + "targetBlank": false, + "title": "Cilium Overviews", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "hubble" + ], + "targetBlank": false, + "title": "Hubble", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "panels": [], + "title": "DNS", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 37, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_dns_queries_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source) > 0", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "DNS queries", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 41, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(hubble_dns_queries_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])*60) by (query))", + "legendFormat": "{{query}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 DNS queries", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 39, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "round(sum(rate(hubble_dns_queries_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source) - sum(label_replace(sum(rate(hubble_dns_responses_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\"}[$__rate_interval])) by (destination), \"source\", \"$1\", \"destination\", \"(.*)\")) without (destination), 0.001) > 0", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing DNS responses", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 43, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_dns_responses_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\", rcode!=\"No Error\"}[$__rate_interval])) by (destination, rcode) > 0", + "legendFormat": "{{destination}}: {{rcode}}", + "range": true, + "refId": "A" + } + ], + "title": "DNS errors", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "kubecon-demo" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "prometheus_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(cilium_version, cluster)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cilium_version, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(source_namespace)", + "hide": 0, + "includeAll": true, + "label": "Source Namespace", + "multi": true, + "name": "source_namespace", + "options": [], + "query": { + "query": "label_values(source_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(destination_namespace)", + "hide": 0, + "includeAll": true, + "label": "Destination Namespace", + "multi": true, + "name": "destination_namespace", + "options": [], + "query": { + "query": "label_values(destination_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Hubble / DNS Overview (Namespace)", + "uid": "_f0DUpY4k", + "version": 26, + "weekStart": "" + } + \ No newline at end of file diff --git a/helm/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json b/helm/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json new file mode 100644 index 00000000..d0cf9d3b --- /dev/null +++ b/helm/cilium/files/hubble/dashboards/hubble-network-overview-namespace.json @@ -0,0 +1,1001 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__elements": {}, + "__requires": [ + { + "type": "panel", + "id": "bargauge", + "name": "Bar gauge", + "version": "" + }, + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "9.4.7" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "timeseries", + "name": "Time series", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 16612, + "graphTooltip": 0, + "id": null, + "links": [ + { + "asDropdown": true, + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "cilium-overview" + ], + "targetBlank": false, + "title": "Cilium Overviews", + "tooltip": "", + "type": "dashboards", + "url": "" + }, + { + "asDropdown": true, + "icon": "external link", + "includeVars": false, + "keepTime": true, + "tags": [ + "hubble" + ], + "targetBlank": false, + "title": "Hubble", + "tooltip": "", + "type": "dashboards", + "url": "" + } + ], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 8, + "panels": [], + "title": "Flows processed", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 12, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (type, subtype)", + "legendFormat": "{{type}}/{{subtype}}", + "range": true, + "refId": "A" + } + ], + "title": "Flows processed by type", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 35, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (verdict)", + "legendFormat": "{{verdict}}", + "range": true, + "refId": "A" + } + ], + "title": "Flows processed by verdict", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 36, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source))", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 sources", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 10 + }, + "id": 37, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "topk(10, sum(rate(hubble_flows_processed_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (destination))", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Top 10 destinations", + "type": "bargauge" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 10, + "panels": [], + "title": "Connection drops", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_tcp_flags_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\", flag=\"SYN\"}[$__rate_interval])) by (source) - sum(label_replace(sum(rate(hubble_tcp_flags_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\", flag=\"SYN-ACK\"}[$__rate_interval])) by (destination), \"source\", \"$1\", \"destination\", \"(.*)\")) without (destination) > 0", + "hide": false, + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing TCP SYN-ACKs", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 20 + }, + "id": 34, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_icmp_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\", type=\"EchoRequest\"}[$__rate_interval])) by (source) - sum(label_replace(sum(rate(hubble_icmp_total{cluster=~\"$cluster\", source_namespace=~\"$destination_namespace\", destination_namespace=~\"$source_namespace\", type=\"EchoReply\"}[$__rate_interval])) by (destination), \"source\", \"$1\", \"destination\", \"(.*)\")) without (destination) > 0", + "legendFormat": "{{source}}", + "range": true, + "refId": "A" + } + ], + "title": "Missing ICMP Echo Replys", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 6, + "panels": [], + "title": "Network Policy drops", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 29, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_drop_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (source, reason) > 0", + "legendFormat": "{{source}}: {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Network Policy drops by source", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "kube-dns-7d44cdb5d5-g85vg: UNSUPPORTED_PROTOCOL_FOR_NAT_MASQUERADE" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 30, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(hubble_drop_total{cluster=~\"$cluster\", source_namespace=~\"$source_namespace\", destination_namespace=~\"$destination_namespace\"}[$__rate_interval])) by (destination, reason) > 0", + "legendFormat": "{{destination}}: {{reason}}", + "range": true, + "refId": "A" + } + ], + "title": "Network Policy drops by destination", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "kubecon-demo" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "Data Source", + "multi": false, + "name": "prometheus_datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "(?!grafanacloud-usage|grafanacloud-ml-metrics).+", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(cilium_version, cluster)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "cluster", + "options": [], + "query": { + "query": "label_values(cilium_version, cluster)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(source_namespace)", + "hide": 0, + "includeAll": true, + "label": "Source Namespace", + "multi": true, + "name": "source_namespace", + "options": [], + "query": { + "query": "label_values(source_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "allValue": ".*", + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(destination_namespace)", + "hide": 0, + "includeAll": true, + "label": "Destination Namespace", + "multi": true, + "name": "destination_namespace", + "options": [], + "query": { + "query": "label_values(destination_namespace)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Hubble / Network Overview (Namespace)", + "uid": "nlsO8tYVz", + "version": 18, + "weekStart": "" + } + \ No newline at end of file diff --git a/helm/cilium/files/nodeinit/startup.bash b/helm/cilium/files/nodeinit/startup.bash index 975b71fa..aaaba092 100644 --- a/helm/cilium/files/nodeinit/startup.bash +++ b/helm/cilium/files/nodeinit/startup.bash @@ -100,7 +100,7 @@ then # Since that version containerd no longer allows missing configuration for the CNI, # not even for pods with hostNetwork set to true. Thus, we add a temporary one. # This will be replaced with the real config by the agent pod. - echo -e "{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}" > /etc/cni/net.d/05-cilium.conf + echo -e '{\n\t"cniVersion": "0.3.1",\n\t"name": "cilium",\n\t"type": "cilium-cni"\n}' > /etc/cni/net.d/05-cilium.conf fi # Start containerd. It won't create it's CNI configuration file anymore. diff --git a/helm/cilium/templates/_helpers.tpl b/helm/cilium/templates/_helpers.tpl index 3e5429e2..b96b3f87 100644 --- a/helm/cilium/templates/_helpers.tpl +++ b/helm/cilium/templates/_helpers.tpl @@ -18,11 +18,20 @@ then `include "cilium.image" .Values.image` will return `quay.io/cilium/cilium:v1.10.1@abcdefgh` */}} {{- define "cilium.image" -}} +{{- if not (kindIs "slice" .) }} +{{- (fail (printf "required list, but got %q" (kindOf .))) }} +{{- end }} +{{- if (ne (len .) 2) }} +{{- (fail (printf "required list of 2 arguments, but got %d" (len .))) }} +{{- end }} +{{- $ := index . 0 }} +{{- with index . 1 }} {{- $digest := (.useDigest | default false) | ternary (printf "@%s" .digest) "" -}} {{- if .override -}} {{- printf "%s" .override -}} {{- else -}} -{{- printf "%s:%s%s" .repository .tag $digest -}} +{{- printf "%s/%s:%s%s" $.Values.image.registry .repository .tag $digest -}} +{{- end -}} {{- end -}} {{- end -}} diff --git a/helm/cilium/templates/cilium-agent/clusterrole.yaml b/helm/cilium/templates/cilium-agent/clusterrole.yaml index 89e604a7..9ccf2192 100644 --- a/helm/cilium/templates/cilium-agent/clusterrole.yaml +++ b/helm/cilium/templates/cilium-agent/clusterrole.yaml @@ -6,6 +6,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -82,6 +86,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -137,6 +144,7 @@ rules: - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch {{- end }} diff --git a/helm/cilium/templates/cilium-agent/clusterrolebinding.yaml b/helm/cilium/templates/cilium-agent/clusterrolebinding.yaml index ecc32441..34b0a58b 100644 --- a/helm/cilium/templates/cilium-agent/clusterrolebinding.yaml +++ b/helm/cilium/templates/cilium-agent/clusterrolebinding.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/cilium-agent/daemonset.yaml b/helm/cilium/templates/cilium-agent/daemonset.yaml index ae6428ba..8b01c66c 100644 --- a/helm/cilium/templates/cilium-agent/daemonset.yaml +++ b/helm/cilium/templates/cilium-agent/daemonset.yaml @@ -16,6 +16,10 @@ kind: DaemonSet metadata: name: cilium namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: cilium app.kubernetes.io/part-of: cilium @@ -83,7 +87,7 @@ spec: {{- end }} containers: - name: cilium-agent - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- if .Values.sleepAfterInit }} command: @@ -124,6 +128,7 @@ spec: failureThreshold: {{ .Values.startupProbe.failureThreshold }} periodSeconds: {{ .Values.startupProbe.periodSeconds }} successThreshold: 1 + initialDelaySeconds: 5 {{- end }} livenessProbe: {{- if or .Values.keepDeprecatedProbes $defaultKeepDeprecatedProbes }} @@ -192,6 +197,10 @@ spec: fieldPath: metadata.namespace - name: CILIUM_CLUSTERMESH_CONFIG value: /var/lib/cilium/clustermesh/ + - name: GOMEMLIMIT + valueFrom: + resourceFieldRef: + resource: limits.memory - name: CILIUM_CNI_CHAINING_MODE valueFrom: configMapKeyRef: @@ -379,12 +388,17 @@ spec: mountPropagation: {{ .mountPropagation }} {{- end }} {{- end }} + {{- if .Values.hubble.export.dynamic.enabled }} + - name: hubble-flowlog-config + mountPath: /flowlog-config + readOnly: true + {{- end }} {{- with .Values.extraVolumeMounts }} {{- toYaml . | nindent 8 }} {{- end }} {{- if .Values.monitor.enabled }} - name: cilium-monitor - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - /bin/bash @@ -395,7 +409,7 @@ spec: for i in {1..5}; do \ [ -S /var/run/cilium/monitor1_2.sock ] && break || sleep 10;\ done; \ - cilium monitor + cilium-dbg monitor {{- range $type := .Values.monitor.eventTypes -}} {{ " " }}--type={{ $type }} {{- end }} @@ -416,10 +430,10 @@ spec: {{- end }} initContainers: - name: config - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - - cilium + - cilium-dbg - build-config {{- with .Values.daemon.configSources }} - "--source={{.}}" @@ -467,12 +481,15 @@ spec: volumeMounts: - name: tmp mountPath: /tmp + {{- with .Values.extraVolumeMounts }} + {{- toYaml . | nindent 8 }} + {{- end }} terminationMessagePolicy: FallbackToLogsOnError {{- if .Values.cgroup.autoMount.enabled }} # Required to mount cgroup2 filesystem on the underlying Kubernetes node. # We use nsenter command with host's cgroup and mount namespaces enabled. - name: mount-cgroup - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} env: - name: CGROUP_ROOT @@ -518,7 +535,7 @@ spec: - ALL {{- end}} - name: apply-sysctl-overwrites - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.initResources }} resources: @@ -567,7 +584,7 @@ spec: # from a privileged container because the mount propagation bidirectional # only works from privileged containers. - name: mount-bpf-fs - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.initResources }} resources: @@ -592,7 +609,7 @@ spec: {{- end }} {{- if and .Values.nodeinit.enabled .Values.nodeinit.bootstrapFile }} - name: wait-for-node-init - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.initResources }} resources: @@ -612,7 +629,7 @@ spec: mountPath: "/tmp/cilium-bootstrap.d" {{- end }} - name: clean-cilium-state - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - /init-container.sh @@ -629,6 +646,12 @@ spec: name: cilium-config key: clean-cilium-bpf-state optional: true + - name: WRITE_CNI_CONF_WHEN_READY + valueFrom: + configMapKeyRef: + name: cilium-config + key: write-cni-conf-when-ready + optional: true {{- if .Values.k8sServiceHost }} - name: KUBERNETES_SERVICE_HOST value: {{ .Values.k8sServiceHost | quote }} @@ -676,9 +699,9 @@ spec: resources: {{- toYaml . | trim | nindent 10 }} {{- end }} - {{- if and .Values.waitForKubeProxy (ne $kubeProxyReplacement "strict") }} + {{- if and .Values.waitForKubeProxy (and (ne $kubeProxyReplacement "strict") (ne $kubeProxyReplacement "true")) }} - name: wait-for-kube-proxy - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} {{- with .Values.initResources }} resources: @@ -715,7 +738,7 @@ spec: {{- end }} # wait-for-kube-proxy {{ if and (.Values.cleanupKubeProxy) (not (eq .Values.kubeProxyReplacement "disabled")) }} - name: cleanup-kube-proxy-iptables - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: "{{ include "cilium.image" (list $ .Values.image) }}" imagePullPolicy: {{ .Values.image.pullPolicy }} securityContext: privileged: true @@ -728,14 +751,14 @@ spec: {{- if .Values.cni.install }} # Install the CNI binaries in an InitContainer so we don't have a writable host mount in the agent - name: install-cni-binaries - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.image }}" + image: {{ include "cilium.image" (list $ .Values.image) | quote }} imagePullPolicy: {{ .Values.image.pullPolicy }} command: - "/install-plugin.sh" + {{- with .Values.cni.resources }} resources: - requests: - cpu: 100m - memory: 10Mi + {{- toYaml . | trim | nindent 10 }} + {{- end }} securityContext: {{- if .Values.securityContext.privileged }} privileged: true @@ -779,7 +802,7 @@ spec: tolerations: {{- toYaml . | trim | nindent 8 }} {{- end }} - {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled (not .Values.clustermesh.apiserver.kvstoremesh.enabled) }} + {{- if and .Values.clustermesh.config.enabled (not (and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.kvstoremesh.enabled )) }} hostAliases: {{- range $cluster := .Values.clustermesh.config.clusters }} {{- range $ip := $cluster.ips }} @@ -973,6 +996,12 @@ spec: path: client-ca.crt {{- end }} {{- end }} + {{- if .Values.hubble.export.dynamic.enabled }} + - name: hubble-flowlog-config + configMap: + name: {{ .Values.hubble.export.dynamic.config.configMapName }} + optional: true + {{- end }} {{- range .Values.extraHostPathMounts }} - name: {{ .name }} hostPath: diff --git a/helm/cilium/templates/cilium-agent/dashboards-configmap.yaml b/helm/cilium/templates/cilium-agent/dashboards-configmap.yaml index 7f1ff7c5..6f19135d 100644 --- a/helm/cilium/templates/cilium-agent/dashboards-configmap.yaml +++ b/helm/cilium/templates/cilium-agent/dashboards-configmap.yaml @@ -15,9 +15,14 @@ metadata: {{- if $.Values.dashboards.label }} {{ $.Values.dashboards.label }}: {{ ternary $.Values.dashboards.labelValue "1" (not (empty $.Values.dashboards.labelValue)) | quote }} {{- end }} - {{- with $.Values.dashboards.annotations }} + {{- if or $.Values.dashboards.annotations $.Values.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with $.Values.dashboards.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} data: {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} diff --git a/helm/cilium/templates/cilium-agent/role.yaml b/helm/cilium/templates/cilium-agent/role.yaml index abbf37d8..e4ad86d1 100644 --- a/helm/cilium/templates/cilium-agent/role.yaml +++ b/helm/cilium/templates/cilium-agent/role.yaml @@ -5,6 +5,10 @@ kind: Role metadata: name: cilium-config-agent namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -26,6 +30,10 @@ kind: Role metadata: name: cilium-ingress-secrets namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -46,6 +54,10 @@ kind: Role metadata: name: cilium-gateway-secrets namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -66,6 +78,30 @@ kind: Role metadata: name: cilium-envoy-config-secrets namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + labels: + app.kubernetes.io/part-of: cilium +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + - watch +{{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.bgpControlPlane.enabled .Values.bgpControlPlane.secretsNamespace.name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: cilium-bgp-control-plane-secrets + namespace: {{ .Values.bgpControlPlane.secretsNamespace.name | quote }} labels: app.kubernetes.io/part-of: cilium rules: diff --git a/helm/cilium/templates/cilium-agent/rolebinding.yaml b/helm/cilium/templates/cilium-agent/rolebinding.yaml index ec8c5d80..cfa8697b 100644 --- a/helm/cilium/templates/cilium-agent/rolebinding.yaml +++ b/helm/cilium/templates/cilium-agent/rolebinding.yaml @@ -5,6 +5,10 @@ kind: RoleBinding metadata: name: cilium-config-agent namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: @@ -24,6 +28,10 @@ kind: RoleBinding metadata: name: cilium-secrets namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: @@ -43,6 +51,10 @@ kind: RoleBinding metadata: name: cilium-gateway-secrets namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: @@ -62,6 +74,10 @@ kind: RoleBinding metadata: name: cilium-envoy-config-secrets namespace: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: @@ -73,3 +89,22 @@ subjects: name: {{ .Values.serviceAccounts.cilium.name | quote }} namespace: {{ .Release.Namespace }} {{- end}} + +{{- if and .Values.agent (not .Values.preflight.enabled) .Values.serviceAccounts.cilium.create .Values.bgpControlPlane.enabled .Values.bgpControlPlane.secretsNamespace.name}} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: cilium-bgp-control-plane-secrets + namespace: {{ .Values.bgpControlPlane.secretsNamespace.name | quote }} + labels: + app.kubernetes.io/part-of: cilium +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: cilium-bgp-control-plane-secrets +subjects: +- kind: ServiceAccount + name: {{ .Values.serviceAccounts.cilium.name | quote }} + namespace: {{ .Release.Namespace }} +{{- end}} diff --git a/helm/cilium/templates/cilium-agent/service.yaml b/helm/cilium/templates/cilium-agent/service.yaml index eedbd3a7..8a34bdee 100644 --- a/helm/cilium/templates/cilium-agent/service.yaml +++ b/helm/cilium/templates/cilium-agent/service.yaml @@ -5,6 +5,10 @@ kind: Service metadata: name: cilium-agent namespace: {{ .Release.Namespace }} + {{- with .Values.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: cilium app.kubernetes.io/name: cilium-agent diff --git a/helm/cilium/templates/cilium-agent/serviceaccount.yaml b/helm/cilium/templates/cilium-agent/serviceaccount.yaml index 605506f1..2c2cf399 100644 --- a/helm/cilium/templates/cilium-agent/serviceaccount.yaml +++ b/helm/cilium/templates/cilium-agent/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.cilium.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.cilium.annotations }} + {{- if or .Values.serviceAccounts.cilium.annotations .Values.annotations }} annotations: - {{- toYaml .Values.serviceAccounts.cilium.annotations | nindent 4 }} + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.cilium.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-agent/servicemonitor.yaml b/helm/cilium/templates/cilium-agent/servicemonitor.yaml index 39a8dcc1..c2ffa66c 100644 --- a/helm/cilium/templates/cilium-agent/servicemonitor.yaml +++ b/helm/cilium/templates/cilium-agent/servicemonitor.yaml @@ -10,10 +10,15 @@ metadata: {{- with .Values.prometheus.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if or .Values.prometheus.serviceMonitor.annotations .Values.annotations }} annotations: + {{- with .Values.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.prometheus.serviceMonitor.annotations }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: selector: matchLabels: @@ -34,6 +39,23 @@ spec: metricRelabelings: {{- toYaml . | nindent 4 }} {{- end }} + {{- if .Values.envoy.prometheus.serviceMonitor.enabled }} + - port: envoy-metrics + interval: {{ .Values.envoy.prometheus.serviceMonitor.interval | quote }} + honorLabels: true + path: /metrics + {{- with .Values.envoy.prometheus.serviceMonitor.relabelings }} + relabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.envoy.prometheus.serviceMonitor.metricRelabelings }} + metricRelabelings: + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} targetLabels: - k8s-app +{{- if .Values.prometheus.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.prometheus.serviceMonitor.jobLabel | quote }} +{{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-ca-secret.yaml b/helm/cilium/templates/cilium-ca-secret.yaml index 5a74f787..f2225b93 100644 --- a/helm/cilium/templates/cilium-ca-secret.yaml +++ b/helm/cilium/templates/cilium-ca-secret.yaml @@ -1,5 +1,5 @@ {{- if or - (and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") (not .Values.clustermesh.apiserver.tls.ca.cert)) + (and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm")) (and (or .Values.agent .Values.hubble.relay.enabled .Values.hubble.ui.enabled) .Values.hubble.enabled .Values.hubble.tls.enabled .Values.hubble.tls.auto.enabled (eq .Values.hubble.tls.auto.method "helm")) (and .Values.tls.ca.key .Values.tls.ca.cert) -}} diff --git a/helm/cilium/templates/cilium-configmap.yaml b/helm/cilium/templates/cilium-configmap.yaml index d06b258c..a3ffbe99 100644 --- a/helm/cilium/templates/cilium-configmap.yaml +++ b/helm/cilium/templates/cilium-configmap.yaml @@ -1,6 +1,5 @@ {{- if and (.Values.agent) (not .Values.preflight.enabled) }} {{- /* Default values with backwards compatibility */ -}} -{{- $defaultEnableCnpStatusUpdates := "true" -}} {{- $defaultBpfMapDynamicSizeRatio := 0.0 -}} {{- $defaultBpfMasquerade := "false" -}} {{- $defaultBpfClockProbe := "false" -}} @@ -13,10 +12,12 @@ {{- $fragmentTracking := "true" -}} {{- $defaultKubeProxyReplacement := "false" -}} {{- $azureUsePrimaryAddress := "true" -}} +{{- $defaultK8sClientQPS := 5 -}} +{{- $defaultK8sClientBurst := 10 -}} +{{- $defaultDNSProxyEnableTransparentMode := "false" -}} {{- /* Default values when 1.8 was initially deployed */ -}} {{- if semverCompare ">=1.8" (default "1.8" .Values.upgradeCompatibility) -}} - {{- $defaultEnableCnpStatusUpdates = "false" -}} {{- $defaultBpfMapDynamicSizeRatio = 0.0025 -}} {{- $defaultBpfMasquerade = "true" -}} {{- $defaultBpfClockProbe = "true" -}} @@ -48,6 +49,7 @@ {{- $azureUsePrimaryAddress = "false" -}} {{- end }} {{- $defaultKubeProxyReplacement = "disabled" -}} + {{- $defaultDNSProxyEnableTransparentMode = "true" -}} {{- end -}} {{- /* Default values when 1.14 was initially deployed */ -}} @@ -76,6 +78,11 @@ {{- else if (not (kindIs "invalid" .Values.cni.chainingTarget)) -}} {{- $cniChainingMode = "generic-veth" -}} {{- end -}} + +{{- if semverCompare ">=1.27-0" .Capabilities.KubeVersion.Version -}} + {{- $defaultK8sClientQPS = 10 -}} + {{- $defaultK8sClientBurst = 20 -}} +{{- end -}} --- apiVersion: v1 kind: ConfigMap @@ -189,6 +196,11 @@ data: enable-policy: "{{ lower .Values.policyEnforcementMode }}" {{- end }} +{{- if hasKey .Values "policyCIDRMatchMode" }} + policy-cidr-match-mode: {{ join " " .Values.policyCIDRMatchMode | quote }} +{{- end}} + + {{- if .Values.prometheus.enabled }} # If you want metrics enabled in all of your Cilium agents, set the port for # which the Cilium agents will have their metrics exposed. @@ -205,6 +217,13 @@ data: {{ . }} {{- end }} {{- end }} + {{- if .Values.prometheus.controllerGroupMetrics }} + # A space-separated list of controller groups for which to enable metrics. + # The special values of "all" and "none" are supported. + controller-group-metrics: {{- range .Values.prometheus.controllerGroupMetrics }} + {{ . }} + {{- end }} + {{- end }} {{- end }} {{- if not .Values.envoy.enabled }} @@ -238,6 +257,7 @@ data: {{- if .Values.ingressController.enabled }} enable-ingress-controller: "true" enforce-ingress-https: {{ .Values.ingressController.enforceHttps | quote }} + enable-ingress-proxy-protocol: {{ .Values.ingressController.enableProxyProtocol | quote }} enable-ingress-secrets-sync: {{ .Values.ingressController.secretsNamespace.sync | quote }} ingress-secrets-namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} ingress-lb-annotation-prefixes: {{ .Values.ingressController.ingressLBAnnotationPrefixes | join " " | quote }} @@ -430,28 +450,23 @@ data: # - vxlan (default) # - geneve {{- if .Values.gke.enabled }} + {{- if ne (.Values.routingMode | default "native") "native" }} + {{- fail (printf "RoutingMode must be set to native when gke.enabled=true" )}} + {{- end }} routing-mode: "native" enable-endpoint-routes: "true" - enable-local-node-route: "false" {{- else if .Values.aksbyocni.enabled }} + {{- if ne (.Values.routingMode | default "tunnel") "tunnel" }} + {{- fail (printf "RoutingMode must be set to tunnel when aksbyocni.enabled=true" )}} + {{- end }} routing-mode: "tunnel" tunnel-protocol: "vxlan" {{- else if .Values.routingMode }} routing-mode: {{ .Values.routingMode | quote }} {{- else }} - {{- if eq .Values.tunnel "disabled" }} - routing-mode: "native" - {{- else if eq .Values.tunnel "vxlan" }} - routing-mode: "tunnel" - tunnel-protocol: "vxlan" - {{- else if eq .Values.tunnel "geneve" }} - routing-mode: "tunnel" - tunnel-protocol: "geneve" - {{- else }} # Default case routing-mode: "tunnel" tunnel-protocol: "vxlan" - {{- end }} {{- end }} {{- if .Values.tunnelProtocol }} @@ -462,6 +477,10 @@ data: tunnel-port: {{ .Values.tunnelPort | quote }} {{- end }} +{{- if .Values.serviceNoBackendResponse }} + service-no-backend-response: "{{ .Values.serviceNoBackendResponse }}" +{{- end}} + {{- if .Values.MTU }} mtu: {{ .Values.MTU | quote }} {{- end }} @@ -500,7 +519,6 @@ data: {{- if .Values.azure.enabled }} enable-endpoint-routes: "true" auto-create-cilium-node-resource: "true" - enable-local-node-route: "false" {{- if .Values.azure.userAssignedIdentityID }} azure-user-assigned-identity-id: {{ .Values.azure.userAssignedIdentityID | quote }} {{- end }} @@ -551,6 +569,7 @@ data: {{- else if eq $defaultBpfMasquerade "true" }} enable-bpf-masquerade: {{ $defaultBpfMasquerade | quote }} {{- end }} + enable-masquerade-to-route-source: {{ .Values.enableMasqueradeRouteSource | quote }} {{- if hasKey .Values "egressMasqueradeInterfaces" }} egress-masquerade-interfaces: {{ .Values.egressMasqueradeInterfaces }} {{- end }} @@ -583,8 +602,8 @@ data: {{- if .Values.encryption.wireguard.userspaceFallback }} enable-wireguard-userspace-fallback: {{ .Values.encryption.wireguard.userspaceFallback | quote }} {{- end }} - {{- if .Values.encryption.wireguard.encapsulate }} - wireguard-encapsulate: {{ .Values.encryption.wireguard.encapsulate | quote }} + {{- if .Values.encryption.wireguard.persistentKeepalive }} + wireguard-persistent-keepalive: {{ .Values.encryption.wireguard.persistentKeepalive | quote }} {{- end }} {{- end }} {{- if .Values.encryption.nodeEncryption }} @@ -592,6 +611,14 @@ data: {{- end }} {{- end }} +{{- if .Values.encryption.strictMode.enabled }} + enable-encryption-strict-mode: {{ .Values.encryption.strictMode.enabled | quote }} + + encryption-strict-mode-cidr: {{ .Values.encryption.strictMode.cidr | quote }} + + encryption-strict-mode-allow-remote-node-identities: {{ .Values.encryption.strictMode.allowRemoteNodeIdentities | quote }} +{{- end }} + enable-xt-socket-fallback: {{ .Values.enableXTSocketFallback | quote }} {{- if or (.Values.azure.enabled) (.Values.eni.enabled) (.Values.gke.enabled) (ne $cniChainingMode "none") }} install-no-conntrack-iptables-rules: "false" @@ -693,6 +720,11 @@ data: {{- end }} {{- if hasKey .Values.nodePort "enableHealthCheck" }} enable-health-check-nodeport: {{ .Values.nodePort.enableHealthCheck | quote}} +{{- end }} +{{- if .Values.gke.enabled }} + enable-health-check-loadbalancer-ip: "true" +{{- else if hasKey .Values.nodePort "enableHealthCheckLoadBalancerIP" }} + enable-health-check-loadbalancer-ip: {{ .Values.nodePort.enableHealthCheckLoadBalancerIP | quote}} {{- end }} node-port-bind-protection: {{ .Values.nodePort.bindProtection | quote }} enable-auto-protect-node-port-range: {{ .Values.nodePort.autoProtectPortRange | quote }} @@ -825,7 +857,7 @@ data: {{- if .Values.hubble.enabled }} # Enable Hubble gRPC service. - enable-hubble: {{ .Values.hubble.enabled | quote }} + enable-hubble: {{ .Values.hubble.enabled | quote }} # UNIX domain socket for Hubble server to listen to. hubble-socket-path: {{ .Values.hubble.socketPath | quote }} {{- if hasKey .Values.hubble "eventQueueSize" }} @@ -849,6 +881,49 @@ data: {{- end }} enable-hubble-open-metrics: {{ .Values.hubble.metrics.enableOpenMetrics | quote }} {{- end }} +{{- if .Values.hubble.redact }} +{{- if eq .Values.hubble.redact.enabled true }} + # Enables hubble redact capabilities + hubble-redact-enabled: "true" +{{- if .Values.hubble.redact.http }} + # Enables redaction of the http URL query part in flows + hubble-redact-http-urlquery: {{ .Values.hubble.redact.http.urlQuery | quote }} + # Enables redaction of the http user info in flows + hubble-redact-http-userinfo: {{ .Values.hubble.redact.http.userInfo | quote }} +{{- if .Values.hubble.redact.http.headers }} +{{- if .Values.hubble.redact.http.headers.allow }} + # Redact all http headers that do not match this list + hubble-redact-http-headers-allow: {{- range .Values.hubble.redact.http.headers.allow }} + {{ . }} +{{- end }} +{{- end }} +{{- if .Values.hubble.redact.http.headers.deny }} + # Redact all http headers that match this list + hubble-redact-http-headers-deny: {{- range .Values.hubble.redact.http.headers.deny }} + {{ . }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hubble.redact.kafka }} + # Enables redaction of the Kafka API key part in flows + hubble-redact-kafka-apikey: {{ .Values.hubble.redact.kafka.apiKey | quote }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hubble.export }} + hubble-export-file-max-size-mb: {{ .Values.hubble.export.fileMaxSizeMb | quote }} + hubble-export-file-max-backups: {{ .Values.hubble.export.fileMaxBackups | quote }} +{{- if .Values.hubble.export.static.enabled }} + hubble-export-file-path: {{ .Values.hubble.export.static.filePath | quote }} + hubble-export-fieldmask: {{ .Values.hubble.export.static.fieldMask | join " " | quote }} + hubble-export-allowlist: {{ .Values.hubble.export.static.allowList | join "," | quote }} + hubble-export-denylist: {{ .Values.hubble.export.static.denyList | join "," | quote }} +{{- end }} +{{- if .Values.hubble.export.dynamic.enabled }} + hubble-flowlogs-config-path: /flowlog-config/flowlogs.yaml +{{- end }} +{{- end }} {{- if hasKey .Values.hubble "listenAddress" }} # An additional address for Hubble server to listen to (e.g. ":4244"). hubble-listen-address: {{ .Values.hubble.listenAddress | quote }} @@ -882,7 +957,7 @@ data: ipam-cilium-node-update-rate: {{ include "validateDuration" .Values.ipam.ciliumNodeUpdateRate | quote }} {{- end }} -{{- if or (eq $ipam "cluster-pool") (eq $ipam "cluster-pool-v2beta") }} +{{- if (eq $ipam "cluster-pool") }} {{- if .Values.ipv4.enabled }} {{- if hasKey .Values.ipam.operator "clusterPoolIPv4PodCIDR" }} {{- /* ipam.operator.clusterPoolIPv4PodCIDR removed in v1.14, remove this failsafe around v1.17 */ -}} @@ -924,11 +999,8 @@ data: limit-ipam-api-qps: {{ .Values.ipam.operator.externalAPILimitQPS | quote }} {{- end }} -{{- if .Values.enableCnpStatusUpdates }} - disable-cnp-status-updates: "false" -{{- else if (eq $defaultEnableCnpStatusUpdates "false") }} - disable-cnp-status-updates: "true" - cnp-node-status-gc-interval: "0s" +{{- if .Values.apiRateLimit }} + api-rate-limit: {{ .Values.apiRateLimit | quote }} {{- end }} {{- if .Values.egressGateway.enabled }} @@ -960,10 +1032,6 @@ data: {{- end }} {{- end }} -{{- if .Values.enableK8sEventHandover }} - enable-k8s-event-handover: "true" -{{- end }} - {{- if .Values.crdWaitTimeout }} crd-wait-timeout: {{ include "validateDuration" .Values.crdWaitTimeout | quote }} {{- end }} @@ -1015,6 +1083,7 @@ data: {{- if .Values.bgpControlPlane.enabled }} enable-bgp-control-plane: "true" + bgp-secrets-namespace: {{ .Values.bgpControlPlane.secretsNamespace.name | quote }} {{- else }} enable-bgp-control-plane: "false" {{- end }} @@ -1061,10 +1130,8 @@ data: annotate-k8s-node: "true" {{- end }} -{{- if hasKey .Values "k8sClientRateLimit" }} - k8s-client-qps: {{ .Values.k8sClientRateLimit.qps | quote }} - k8s-client-burst: {{ .Values.k8sClientRateLimit.burst | quote }} -{{- end }} + k8s-client-qps: {{ .Values.k8sClientRateLimit.qps | default $defaultK8sClientQPS | quote}} + k8s-client-burst: {{ .Values.k8sClientRateLimit.burst | default $defaultK8sClientBurst | quote }} {{- if and .Values.operator.setNodeTaints (not .Values.operator.removeNodeTaints) -}} {{ fail "Cannot have operator.setNodeTaintsMaxNodes and not operator.removeNodeTaints = false" }} @@ -1089,6 +1156,13 @@ data: {{- end }} {{- if .Values.dnsProxy }} + {{- if hasKey .Values.dnsProxy "enableTransparentMode" }} + # explicit setting gets precedence + dnsproxy-enable-transparent-mode: {{ .Values.dnsProxy.enableTransparentMode | quote }} + {{- else if eq $cniChainingMode "none" }} + # default DNS proxy to transparent mode in non-chaining modes + dnsproxy-enable-transparent-mode: {{ $defaultDNSProxyEnableTransparentMode | quote }} + {{- end }} {{- if .Values.dnsProxy.dnsRejectResponseCode }} tofqdns-dns-reject-response-code: {{ .Values.dnsProxy.dnsRejectResponseCode | quote }} {{- end }} @@ -1118,10 +1192,6 @@ data: {{- end }} {{- end }} -{{- if .Values.extraConfig }} - {{ toYaml .Values.extraConfig | nindent 2 }} -{{- end }} - {{- if hasKey .Values "agentNotReadyTaintKey" }} agent-not-ready-taint-key: {{ .Values.agentNotReadyTaintKey | quote }} {{- end }} @@ -1135,6 +1205,7 @@ data: mesh-auth-mutual-enabled: "true" mesh-auth-mutual-listener-port: {{ .Values.authentication.mutual.port | quote }} mesh-auth-spire-agent-socket: {{ .Values.authentication.mutual.spire.agentSocketPath | quote }} + mesh-auth-mutual-connect-timeout: {{ include "validateDuration" .Values.authentication.mutual.connectTimeout | quote }} {{- if .Values.authentication.mutual.spire.serverAddress }} mesh-auth-spire-server-address: {{ .Values.authentication.mutual.spire.serverAddress | quote }} {{- else }} @@ -1155,6 +1226,16 @@ data: envoy-log: {{ .Values.envoy.log.path | quote }} {{- end }} +{{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + max-connected-clusters: {{ .Values.clustermesh.maxConnectedClusters | quote }} +{{- end }} + +# Extra config allows adding arbitrary properties to the cilium config. +# By putting it at the end of the ConfigMap, it's also possible to override existing properties. +{{- if .Values.extraConfig }} + {{ toYaml .Values.extraConfig | nindent 2 }} +{{- end }} + {{- end }} --- {{- if and .Values.ipMasqAgent.enabled .Values.ipMasqAgent.config }} diff --git a/helm/cilium/templates/cilium-envoy/configmap.yaml b/helm/cilium/templates/cilium-envoy/configmap.yaml index 7aa0f046..990cf951 100644 --- a/helm/cilium/templates/cilium-envoy/configmap.yaml +++ b/helm/cilium/templates/cilium-envoy/configmap.yaml @@ -6,6 +6,10 @@ kind: ConfigMap metadata: name: cilium-envoy-config namespace: {{ .Release.Namespace }} + {{- with .Values.envoy.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: {{- (tpl (.Files.Glob "files/cilium-envoy/configmap/bootstrap-config.json").AsConfig .) | nindent 2 }} diff --git a/helm/cilium/templates/cilium-envoy/daemonset.yaml b/helm/cilium/templates/cilium-envoy/daemonset.yaml index 2ff9692e..d6d43688 100644 --- a/helm/cilium/templates/cilium-envoy/daemonset.yaml +++ b/helm/cilium/templates/cilium-envoy/daemonset.yaml @@ -6,6 +6,10 @@ kind: DaemonSet metadata: name: cilium-envoy namespace: {{ .Release.Namespace }} + {{- with .Values.envoy.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: cilium-envoy app.kubernetes.io/part-of: cilium @@ -58,14 +62,14 @@ spec: {{- end }} containers: - name: cilium-envoy - image: {{ include "cilium.image" .Values.envoy.image | quote }} + image: {{ include "cilium.image" (list $ .Values.envoy.image) | quote }} imagePullPolicy: {{ .Values.envoy.image.pullPolicy }} command: - - /usr/bin/cilium-envoy + - /usr/bin/cilium-envoy-starter args: - '-c /var/run/cilium/envoy/bootstrap-config.json' - '--base-id 0' - {{- if and (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "envoy" ( splitList " " .Values.debug.verbose )) }} + {{- if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "envoy" ( splitList " " .Values.debug.verbose )) }} - '--log-level trace' {{- else if and (.Values.debug.enabled) (hasKey .Values.debug "verbose") (.Values.debug.verbose) (has "flow" ( splitList " " .Values.debug.verbose )) }} - '--log-level debug' @@ -89,6 +93,7 @@ spec: failureThreshold: {{ .Values.envoy.startupProbe.failureThreshold }} periodSeconds: {{ .Values.envoy.startupProbe.periodSeconds }} successThreshold: 1 + initialDelaySeconds: 5 {{- end }} livenessProbe: httpGet: @@ -175,6 +180,9 @@ spec: - name: envoy-sockets mountPath: /var/run/cilium/envoy/sockets readOnly: false + - name: envoy-artifacts + mountPath: /var/run/cilium/envoy/artifacts + readOnly: true - name: envoy-config mountPath: /var/run/cilium/envoy/ readOnly: true @@ -224,6 +232,10 @@ spec: hostPath: path: "{{ .Values.daemon.runPath }}/envoy/sockets" type: DirectoryOrCreate + - name: envoy-artifacts + hostPath: + path: "{{ .Values.daemon.runPath }}/envoy/artifacts" + type: DirectoryOrCreate - name: envoy-config configMap: name: cilium-envoy-config diff --git a/helm/cilium/templates/cilium-envoy/service.yaml b/helm/cilium/templates/cilium-envoy/service.yaml index f5c2a09b..d238c62e 100644 --- a/helm/cilium/templates/cilium-envoy/service.yaml +++ b/helm/cilium/templates/cilium-envoy/service.yaml @@ -4,11 +4,16 @@ kind: Service metadata: name: cilium-envoy namespace: {{ .Release.Namespace }} - {{- if not .Values.envoy.prometheus.serviceMonitor.enabled }} + {{- if or (not .Values.envoy.prometheus.serviceMonitor.enabled) .Values.envoy.annotations }} annotations: + {{- if not .Values.envoy.prometheus.serviceMonitor.enabled }} prometheus.io/scrape: "true" prometheus.io/port: {{ .Values.proxy.prometheus.port | default .Values.envoy.prometheus.port | quote }} {{- end }} + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- end }} labels: k8s-app: cilium-envoy app.kubernetes.io/name: cilium-envoy diff --git a/helm/cilium/templates/cilium-envoy/serviceaccount.yaml b/helm/cilium/templates/cilium-envoy/serviceaccount.yaml index 6094ba20..607dabca 100644 --- a/helm/cilium/templates/cilium-envoy/serviceaccount.yaml +++ b/helm/cilium/templates/cilium-envoy/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.envoy.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.envoy.annotations }} + {{- if or .Values.serviceAccounts.envoy.annotations .Values.envoy.annotations }} annotations: - {{- toYaml .Values.serviceAccounts.envoy.annotations | nindent 4 }} + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-envoy/servicemonitor.yaml b/helm/cilium/templates/cilium-envoy/servicemonitor.yaml index 363b3776..10f84d82 100644 --- a/helm/cilium/templates/cilium-envoy/servicemonitor.yaml +++ b/helm/cilium/templates/cilium-envoy/servicemonitor.yaml @@ -7,13 +7,19 @@ metadata: namespace: {{ .Values.envoy.prometheus.serviceMonitor.namespace | default .Release.Namespace }} labels: app.kubernetes.io/part-of: cilium + app.kubernetes.io/name: cilium-envoy {{- with .Values.envoy.prometheus.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if or .Values.envoy.prometheus.serviceMonitor.annotations .Values.envoy.annotations }} annotations: + {{- with .Values.envoy.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.envoy.prometheus.serviceMonitor.annotations }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: selector: matchLabels: @@ -22,7 +28,7 @@ spec: matchNames: - {{ .Release.Namespace }} endpoints: - - port: metrics + - port: envoy-metrics interval: {{ .Values.envoy.prometheus.serviceMonitor.interval | quote }} honorLabels: true path: /metrics diff --git a/helm/cilium/templates/cilium-flowlog-configmap.yaml b/helm/cilium/templates/cilium-flowlog-configmap.yaml new file mode 100644 index 00000000..8a1341e4 --- /dev/null +++ b/helm/cilium/templates/cilium-flowlog-configmap.yaml @@ -0,0 +1,12 @@ +{{- if and .Values.hubble.export.dynamic.enabled .Values.hubble.export.dynamic.config.createConfigMap }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Values.hubble.export.dynamic.config.configMapName }} + namespace: {{ .Release.Namespace }} +data: + flowlogs.yaml: | + flowLogs: +{{ .Values.hubble.export.dynamic.config.content | toYaml | indent 4 }} +{{- end }} diff --git a/helm/cilium/templates/cilium-gateway-api-class.yaml b/helm/cilium/templates/cilium-gateway-api-class.yaml index 089649ff..74a78d8d 100644 --- a/helm/cilium/templates/cilium-gateway-api-class.yaml +++ b/helm/cilium/templates/cilium-gateway-api-class.yaml @@ -1,6 +1,6 @@ {{- if .Values.gatewayAPI.enabled -}} -{{- if .Capabilities.APIVersions.Has "gateway.networking.k8s.io/v1beta1/GatewayClass" }} -apiVersion: gateway.networking.k8s.io/v1beta1 +{{- if .Capabilities.APIVersions.Has "gateway.networking.k8s.io/v1/GatewayClass" }} +apiVersion: gateway.networking.k8s.io/v1 kind: GatewayClass metadata: name: cilium diff --git a/helm/cilium/templates/cilium-nodeinit/daemonset.yaml b/helm/cilium/templates/cilium-nodeinit/daemonset.yaml index 65b05f03..56b552ac 100644 --- a/helm/cilium/templates/cilium-nodeinit/daemonset.yaml +++ b/helm/cilium/templates/cilium-nodeinit/daemonset.yaml @@ -5,6 +5,10 @@ apiVersion: apps/v1 metadata: name: cilium-node-init namespace: {{ .Release.Namespace }} + {{- with .Values.nodeinit.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app: cilium-node-init app.kubernetes.io/part-of: cilium @@ -43,7 +47,7 @@ spec: {{- end }} containers: - name: node-init - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.nodeinit.image }}" + image: {{ include "cilium.image" (list $ .Values.nodeinit.image) | quote }} imagePullPolicy: {{ .Values.nodeinit.image.pullPolicy }} lifecycle: {{- if .Values.nodeinit.revertReconfigureKubelet }} diff --git a/helm/cilium/templates/cilium-nodeinit/serviceaccount.yaml b/helm/cilium/templates/cilium-nodeinit/serviceaccount.yaml index 806e3706..eb503ee9 100644 --- a/helm/cilium/templates/cilium-nodeinit/serviceaccount.yaml +++ b/helm/cilium/templates/cilium-nodeinit/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.nodeinit.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.nodeinit.annotations }} + {{- if or .Values.serviceAccounts.nodeinit.annotations .Values.nodeinit.annotations }} annotations: - {{- toYaml .Values.serviceAccounts.nodeinit.annotations | nindent 4 }} + {{- with .Values.nodeinit.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.nodeinit.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-operator/_helpers.tpl b/helm/cilium/templates/cilium-operator/_helpers.tpl index 7bf67f59..bcc1cf20 100644 --- a/helm/cilium/templates/cilium-operator/_helpers.tpl +++ b/helm/cilium/templates/cilium-operator/_helpers.tpl @@ -26,11 +26,20 @@ Return cilium operator image */}} {{- define "cilium.operator.image" -}} +{{- if not (kindIs "slice" .) }} +{{- (fail (printf "required list, but got %q" (kindOf .))) }} +{{- end }} +{{- if (ne (len .) 2) }} +{{- (fail (printf "required list of 2 arguments, but got %d" (len .))) }} +{{- end }} +{{- $ := index . 0 }} +{{- with index . 1 }} {{- if .Values.operator.image.override -}} {{- printf "%s" .Values.operator.image.override -}} {{- else -}} {{- $cloud := include "cilium.operator.cloud" . }} {{- $imageDigest := include "cilium.operator.imageDigestName" . }} -{{- printf "%s/%s-%s%s:%s%s" .Values.image.registry .Values.operator.image.repository $cloud .Values.operator.image.suffix .Values.operator.image.tag $imageDigest -}} +{{- printf "%s/%s-%s%s:%s%s" $.Values.image.registry .Values.operator.image.repository $cloud .Values.operator.image.suffix .Values.operator.image.tag $imageDigest -}} +{{- end -}} {{- end -}} {{- end -}} diff --git a/helm/cilium/templates/cilium-operator/clusterrole.yaml b/helm/cilium/templates/cilium-operator/clusterrole.yaml index 862fac10..7e9080b5 100644 --- a/helm/cilium/templates/cilium-operator/clusterrole.yaml +++ b/helm/cilium/templates/cilium-operator/clusterrole.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium-operator + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -157,6 +161,9 @@ rules: resources: - ciliumendpointslices - ciliumenvoyconfigs + - ciliumbgppeerconfigs + - ciliumbgpadvertisements + - ciliumbgpnodeconfigs verbs: - create - update @@ -183,6 +190,11 @@ rules: resourceNames: - ciliumloadbalancerippools.cilium.io - ciliumbgppeeringpolicies.cilium.io + - ciliumbgpclusterconfigs.cilium.io + - ciliumbgppeerconfigs.cilium.io + - ciliumbgpadvertisements.cilium.io + - ciliumbgpnodeconfigs.cilium.io + - ciliumbgpnodeconfigoverrides.cilium.io - ciliumclusterwideenvoyconfigs.cilium.io - ciliumclusterwidenetworkpolicies.cilium.io - ciliumegressgatewaypolicies.cilium.io @@ -203,6 +215,8 @@ rules: resources: - ciliumloadbalancerippools - ciliumpodippools + - ciliumbgpclusterconfigs + - ciliumbgpnodeconfigoverrides verbs: - get - list @@ -258,6 +272,7 @@ rules: - gateways - tlsroutes - httproutes + - grpcroutes - referencegrants - referencepolicies verbs: @@ -270,6 +285,7 @@ rules: - gatewayclasses/status - gateways/status - httproutes/status + - grpcroutes/status - tlsroutes/status verbs: - update diff --git a/helm/cilium/templates/cilium-operator/clusterrolebinding.yaml b/helm/cilium/templates/cilium-operator/clusterrolebinding.yaml index a8a51bd4..f0d00e2e 100644 --- a/helm/cilium/templates/cilium-operator/clusterrolebinding.yaml +++ b/helm/cilium/templates/cilium-operator/clusterrolebinding.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium-operator + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/cilium-operator/dashboards-configmap.yaml b/helm/cilium/templates/cilium-operator/dashboards-configmap.yaml index 37402bed..c4b90a27 100644 --- a/helm/cilium/templates/cilium-operator/dashboards-configmap.yaml +++ b/helm/cilium/templates/cilium-operator/dashboards-configmap.yaml @@ -15,9 +15,14 @@ metadata: {{- if $.Values.operator.dashboards.label }} {{ $.Values.operator.dashboards.label }}: {{ ternary $.Values.operator.dashboards.labelValue "1" (not (empty $.Values.operator.dashboards.labelValue)) | quote }} {{- end }} - {{- with $.Values.operator.dashboards.annotations }} + {{- if or $.Values.operator.dashboards.annotations $.Values.operator.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with $.Values.operator.dashboards.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} data: {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} diff --git a/helm/cilium/templates/cilium-operator/deployment.yaml b/helm/cilium/templates/cilium-operator/deployment.yaml index 44b1da2a..cd73c8a6 100644 --- a/helm/cilium/templates/cilium-operator/deployment.yaml +++ b/helm/cilium/templates/cilium-operator/deployment.yaml @@ -5,6 +5,10 @@ kind: Deployment metadata: name: cilium-operator namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: io.cilium/app: operator name: cilium-operator @@ -67,7 +71,7 @@ spec: {{- end }} containers: - name: cilium-operator - image: {{ include "cilium.operator.image" . | quote }} + image: {{ include "cilium.operator.image" (list $ .) | quote }} imagePullPolicy: {{ .Values.operator.image.pullPolicy }} command: - cilium-operator-{{ include "cilium.operator.cloud" . }} diff --git a/helm/cilium/templates/cilium-operator/poddisruptionbudget.yaml b/helm/cilium/templates/cilium-operator/poddisruptionbudget.yaml index cb63aaf0..a224b9e6 100644 --- a/helm/cilium/templates/cilium-operator/poddisruptionbudget.yaml +++ b/helm/cilium/templates/cilium-operator/poddisruptionbudget.yaml @@ -5,6 +5,10 @@ kind: PodDisruptionBudget metadata: name: cilium-operator namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: io.cilium/app: operator name: cilium-operator diff --git a/helm/cilium/templates/cilium-operator/role.yaml b/helm/cilium/templates/cilium-operator/role.yaml index cf2526d9..20101b02 100644 --- a/helm/cilium/templates/cilium-operator/role.yaml +++ b/helm/cilium/templates/cilium-operator/role.yaml @@ -5,6 +5,10 @@ kind: Role metadata: name: cilium-operator-ingress-secrets namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -26,6 +30,10 @@ kind: Role metadata: name: cilium-operator-gateway-secrets namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: diff --git a/helm/cilium/templates/cilium-operator/rolebinding.yaml b/helm/cilium/templates/cilium-operator/rolebinding.yaml index aa826906..8d8f73d8 100644 --- a/helm/cilium/templates/cilium-operator/rolebinding.yaml +++ b/helm/cilium/templates/cilium-operator/rolebinding.yaml @@ -5,6 +5,10 @@ kind: RoleBinding metadata: name: cilium-operator-ingress-secrets namespace: {{ .Values.ingressController.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: @@ -24,6 +28,10 @@ kind: RoleBinding metadata: name: cilium-operator-gateway-secrets namespace: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/cilium-operator/secret.yaml b/helm/cilium/templates/cilium-operator/secret.yaml index 42096412..31e917ec 100644 --- a/helm/cilium/templates/cilium-operator/secret.yaml +++ b/helm/cilium/templates/cilium-operator/secret.yaml @@ -5,6 +5,10 @@ kind: Secret metadata: name: cilium-azure namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: Opaque data: AZURE_CLIENT_ID: {{ default "" .Values.azure.clientID | b64enc | quote }} diff --git a/helm/cilium/templates/cilium-operator/service.yaml b/helm/cilium/templates/cilium-operator/service.yaml index 61514916..8c201099 100644 --- a/helm/cilium/templates/cilium-operator/service.yaml +++ b/helm/cilium/templates/cilium-operator/service.yaml @@ -4,6 +4,10 @@ apiVersion: v1 metadata: name: cilium-operator namespace: {{ .Release.Namespace }} + {{- with .Values.operator.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: io.cilium/app: operator name: cilium-operator diff --git a/helm/cilium/templates/cilium-operator/serviceaccount.yaml b/helm/cilium/templates/cilium-operator/serviceaccount.yaml index 4bb3551c..b083907e 100644 --- a/helm/cilium/templates/cilium-operator/serviceaccount.yaml +++ b/helm/cilium/templates/cilium-operator/serviceaccount.yaml @@ -8,8 +8,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.operator.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.operator.annotations }} + {{- if or .Values.serviceAccounts.operator.annotations .Values.operator.annotations }} annotations: - {{- toYaml .Values.serviceAccounts.operator.annotations | nindent 4 }} + {{- with .Values.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-operator/servicemonitor.yaml b/helm/cilium/templates/cilium-operator/servicemonitor.yaml index a3edaec5..742993b8 100644 --- a/helm/cilium/templates/cilium-operator/servicemonitor.yaml +++ b/helm/cilium/templates/cilium-operator/servicemonitor.yaml @@ -10,10 +10,15 @@ metadata: {{- with .Values.operator.prometheus.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if or .Values.operator.prometheus.serviceMonitor.annotations .Values.operator.annotations }} annotations: + {{- with .Values.operator.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.operator.prometheus.serviceMonitor.annotations }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: selector: matchLabels: @@ -37,4 +42,7 @@ spec: {{- end }} targetLabels: - io.cilium/app +{{- if .Values.operator.prometheus.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.operator.prometheus.serviceMonitor.jobLabel | quote }} +{{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-preflight/clusterrole.yaml b/helm/cilium/templates/cilium-preflight/clusterrole.yaml index 379814c8..d2a8f4ca 100644 --- a/helm/cilium/templates/cilium-preflight/clusterrole.yaml +++ b/helm/cilium/templates/cilium-preflight/clusterrole.yaml @@ -6,6 +6,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium-pre-flight + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -82,6 +86,9 @@ rules: resources: - ciliumloadbalancerippools - ciliumbgppeeringpolicies + - ciliumbgpnodeconfigs + - ciliumbgpadvertisements + - ciliumbgppeerconfigs - ciliumclusterwideenvoyconfigs - ciliumclusterwidenetworkpolicies - ciliumegressgatewaypolicies @@ -137,6 +144,7 @@ rules: - ciliumendpoints/status - ciliumendpoints - ciliuml2announcementpolicies/status + - ciliumbgpnodeconfigs/status verbs: - patch {{- end }} diff --git a/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml b/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml index ef5be1de..1237d6ac 100644 --- a/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml +++ b/helm/cilium/templates/cilium-preflight/clusterrolebinding.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium-pre-flight + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/cilium-preflight/daemonset.yaml b/helm/cilium/templates/cilium-preflight/daemonset.yaml index 31f57b1d..c145abcb 100644 --- a/helm/cilium/templates/cilium-preflight/daemonset.yaml +++ b/helm/cilium/templates/cilium-preflight/daemonset.yaml @@ -4,6 +4,10 @@ kind: DaemonSet metadata: name: cilium-pre-flight-check namespace: {{ .Release.Namespace }} + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: selector: matchLabels: @@ -38,7 +42,7 @@ spec: {{- end }} initContainers: - name: clean-cilium-state - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.preflight.image }}" + image: {{ include "cilium.image" (list $ .Values.preflight.image) | quote }} imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} command: ["/bin/echo"] args: @@ -46,7 +50,7 @@ spec: terminationMessagePolicy: FallbackToLogsOnError containers: - name: cilium-pre-flight-check - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.preflight.image }}" + image: {{ include "cilium.image" (list $ .Values.preflight.image) | quote }} imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} command: ["/bin/sh"] args: @@ -66,8 +70,13 @@ spec: - /tmp/ready initialDelaySeconds: 5 periodSeconds: 5 - {{- with .Values.preflight.extraEnv }} env: + - name: K8S_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + {{- with .Values.preflight.extraEnv }} {{- toYaml . | trim | nindent 12 }} {{- end }} volumeMounts: @@ -97,14 +106,14 @@ spec: terminationMessagePolicy: FallbackToLogsOnError {{- if ne .Values.preflight.tofqdnsPreCache "" }} - name: cilium-pre-flight-fqdn-precache - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.preflight.image }}" + image: {{ include "cilium.image" (list $ .Values.preflight.image) | quote }} imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} name: cilium-pre-flight-fqdn-precache command: ["/bin/sh"] args: - -ec - | - cilium preflight fqdn-poller --tofqdns-pre-cache {{ .Values.preflight.tofqdnsPreCache }}; + cilium-dbg preflight fqdn-poller --tofqdns-pre-cache {{ .Values.preflight.tofqdnsPreCache }}; touch /tmp/ready-tofqdns-precache; livenessProbe: exec: diff --git a/helm/cilium/templates/cilium-preflight/deployment.yaml b/helm/cilium/templates/cilium-preflight/deployment.yaml index b1283c59..9c259d41 100644 --- a/helm/cilium/templates/cilium-preflight/deployment.yaml +++ b/helm/cilium/templates/cilium-preflight/deployment.yaml @@ -4,6 +4,10 @@ kind: Deployment metadata: name: cilium-pre-flight-check namespace: {{ .Release.Namespace }} + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium app.kubernetes.io/name: cilium-pre-flight-check @@ -33,13 +37,13 @@ spec: {{- end }} containers: - name: cnp-validator - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.preflight.image }}" + image: {{ include "cilium.image" (list $ .Values.preflight.image) | quote }} imagePullPolicy: {{ .Values.preflight.image.pullPolicy }} command: ["/bin/sh"] args: - -ec - | - cilium preflight validate-cnp; + cilium-dbg preflight validate-cnp; touch /tmp/ready-validate-cnp; sleep 1h; livenessProbe: diff --git a/helm/cilium/templates/cilium-preflight/poddisruptionbudget.yaml b/helm/cilium/templates/cilium-preflight/poddisruptionbudget.yaml index d939a9a2..4b3c7cb0 100644 --- a/helm/cilium/templates/cilium-preflight/poddisruptionbudget.yaml +++ b/helm/cilium/templates/cilium-preflight/poddisruptionbudget.yaml @@ -5,6 +5,10 @@ kind: PodDisruptionBudget metadata: name: cilium-pre-flight-check namespace: {{ .Release.Namespace }} + {{- with .Values.preflight.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: cilium-pre-flight-check-deployment app.kubernetes.io/part-of: cilium diff --git a/helm/cilium/templates/cilium-preflight/serviceaccount.yaml b/helm/cilium/templates/cilium-preflight/serviceaccount.yaml index a55aaa40..b64a3296 100644 --- a/helm/cilium/templates/cilium-preflight/serviceaccount.yaml +++ b/helm/cilium/templates/cilium-preflight/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.preflight.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.preflight.annotations }} + {{- if or .Values.serviceAccounts.preflight.annotations .Values.preflight.annotations }} annotations: - {{ toYaml .Values.serviceAccounts.preflight.annotations | nindent 4 }} + {{- with .Values.preflight.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.preflight.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/cilium-secrets-namespace.yaml b/helm/cilium/templates/cilium-secrets-namespace.yaml index 6e1aa43d..d99967fa 100644 --- a/helm/cilium/templates/cilium-secrets-namespace.yaml +++ b/helm/cilium/templates/cilium-secrets-namespace.yaml @@ -1,32 +1,14 @@ -{{- if and .Values.ingressController.enabled .Values.ingressController.secretsNamespace.create .Values.ingressController.secretsNamespace.name }} ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Values.ingressController.secretsNamespace.name | quote }} -{{- end}} - -# Only create the namespace if it's different from Ingress secret namespace or Ingress is not enabled. -{{- if and .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.create .Values.gatewayAPI.secretsNamespace.name - (or (not (and .Values.ingressController.enabled .Values.ingressController.secretsNamespace.create .Values.ingressController.secretsNamespace.name)) - (ne .Values.gatewayAPI.secretsNamespace.name .Values.ingressController.secretsNamespace.name)) }} ---- -apiVersion: v1 -kind: Namespace -metadata: - name: {{ .Values.gatewayAPI.secretsNamespace.name | quote }} -{{- end}} +{{- $secretNamespaces := dict -}} +{{- range $cfg := tuple .Values.ingressController .Values.gatewayAPI .Values.envoyConfig .Values.bgpControlPlane -}} +{{- if and $cfg.enabled $cfg.secretsNamespace.create $cfg.secretsNamespace.name -}} +{{- $_ := set $secretNamespaces $cfg.secretsNamespace.name 1 -}} +{{- end -}} +{{- end -}} -# Only create the namespace if it's different from Ingress and Gateway API secret namespaces (if enabled). -{{- if and .Values.envoyConfig.enabled .Values.envoyConfig.secretsNamespace.create .Values.envoyConfig.secretsNamespace.name - (and - (or (not (and .Values.ingressController.enabled .Values.ingressController.secretsNamespace.create .Values.ingressController.secretsNamespace.name)) - (ne .Values.envoyConfig.secretsNamespace.name .Values.ingressController.secretsNamespace.name)) - (or (not (and .Values.gatewayAPI.enabled .Values.gatewayAPI.secretsNamespace.create .Values.gatewayAPI.secretsNamespace.name)) - (ne .Values.envoyConfig.secretsNamespace.name .Values.gatewayAPI.secretsNamespace.name))) }} +{{- range $name, $_ := $secretNamespaces }} --- apiVersion: v1 kind: Namespace metadata: - name: {{ .Values.envoyConfig.secretsNamespace.name | quote }} + name: {{ $name | quote }} {{- end}} diff --git a/helm/cilium/templates/clustermesh-apiserver/clusterrole.yaml b/helm/cilium/templates/clustermesh-apiserver/clusterrole.yaml index c00c8a51..c5df0163 100644 --- a/helm/cilium/templates/clustermesh-apiserver/clusterrole.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/clusterrole.yaml @@ -5,6 +5,10 @@ metadata: name: clustermesh-apiserver labels: app.kubernetes.io/part-of: cilium + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} rules: - apiGroups: - cilium.io diff --git a/helm/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml b/helm/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml index 8ef03cb2..edc3cf9c 100644 --- a/helm/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/clusterrolebinding.yaml @@ -5,6 +5,10 @@ metadata: name: clustermesh-apiserver labels: app.kubernetes.io/part-of: cilium + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/helm/cilium/templates/clustermesh-apiserver/deployment.yaml b/helm/cilium/templates/clustermesh-apiserver/deployment.yaml index 969fb6fc..e972fba6 100644 --- a/helm/cilium/templates/clustermesh-apiserver/deployment.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/deployment.yaml @@ -7,6 +7,10 @@ kind: Deployment metadata: name: clustermesh-apiserver namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: clustermesh-apiserver app.kubernetes.io/part-of: cilium @@ -44,41 +48,37 @@ spec: {{- end }} initContainers: - name: etcd-init - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.clustermesh.apiserver.etcd.image }}" - imagePullPolicy: {{ .Values.clustermesh.apiserver.etcd.image.pullPolicy }} - command: ["/bin/sh", "-c"] + image: {{ include "cilium.image" (list $ .Values.clustermesh.apiserver.image) | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} + command: + - /usr/bin/clustermesh-apiserver args: - - | - rm -rf /var/run/etcd/*; - /usr/local/bin/etcd --data-dir=/var/run/etcd --name=clustermesh-apiserver --listen-client-urls=http://127.0.0.1:2379 --advertise-client-urls=http://127.0.0.1:2379 --initial-cluster-token=clustermesh-apiserver --initial-cluster-state=new --auto-compaction-retention=1 & - - # The following key needs to be created before that the cilium agents - # have the possibility of connecting to etcd. - etcdctl put cilium/.has-cluster-config true - - etcdctl user add root --no-password; - etcdctl user grant-role root root; - etcdctl user add admin-{{ .Values.cluster.name }} --no-password; - etcdctl user grant-role admin-{{ .Values.cluster.name }} root; - etcdctl user add externalworkload --no-password; - etcdctl role add externalworkload; - etcdctl role grant-permission externalworkload --from-key read ''; - etcdctl role grant-permission externalworkload readwrite --prefix cilium/state/noderegister/v1/; - etcdctl role grant-permission externalworkload readwrite --prefix cilium/.initlock/; - etcdctl user grant-role externalworkload externalworkload; - etcdctl user add remote --no-password; - etcdctl role add remote; - etcdctl role grant-permission remote --from-key read ''; - etcdctl user grant-role remote remote; - etcdctl auth enable; - exit + - etcdinit + {{- if .Values.debug.enabled }} + - --debug + {{- end }} + # These need to match the equivalent arguments to etcd in the main container. + - --etcd-cluster-name=clustermesh-apiserver + - --etcd-initial-cluster-token=clustermesh-apiserver + - --etcd-data-dir=/var/run/etcd + {{- with .Values.clustermesh.apiserver.etcd.init.extraArgs }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} env: - - name: ETCDCTL_API - value: "3" - - name: HOSTNAME_IP + # The Cilium cluster name (specified via the `CILIUM_CLUSTER_NAME` environment variable) and the etcd cluster + # name (specified via the `--etcd-cluster-name` argument) are very different concepts. The Cilium cluster name + # is the name of the overall Cilium cluster, and is used to set the admin account username. The etcd cluster + # name is a concept that's only relevant for etcd itself. The etcd cluster name must be the same for both this + # command and the actual invocation of etcd in the main containers of this Pod, but it's otherwise not + # relevant to Cilium. + - name: CILIUM_CLUSTER_NAME valueFrom: - fieldRef: - fieldPath: status.podIP + configMapKeyRef: + name: cilium-config + key: cluster-name + {{- with .Values.clustermesh.apiserver.etcd.init.extraEnv }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} volumeMounts: - name: etcd-data-dir mountPath: /var/run/etcd @@ -92,10 +92,11 @@ spec: {{- end }} containers: - name: etcd - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.clustermesh.apiserver.etcd.image }}" - imagePullPolicy: {{ .Values.clustermesh.apiserver.etcd.image.pullPolicy }} + # The clustermesh-apiserver container image includes an etcd binary. + image: {{ include "cilium.image" (list $ .Values.clustermesh.apiserver.image) | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} command: - - /usr/local/bin/etcd + - /usr/bin/etcd args: - --data-dir=/var/run/etcd - --name=clustermesh-apiserver @@ -147,12 +148,17 @@ spec: securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.clustermesh.apiserver.etcd.lifecycle }} + lifecycle: + {{- toYaml . | nindent 10 }} + {{- end }} - name: apiserver - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.clustermesh.apiserver.image }}" + image: {{ include "cilium.image" (list $ .Values.clustermesh.apiserver.image) | quote }} imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} command: - /usr/bin/clustermesh-apiserver args: + - clustermesh {{- if .Values.debug.enabled }} - --debug {{- end }} @@ -160,6 +166,9 @@ spec: - --cluster-id=$(CLUSTER_ID) - --kvstore-opt - etcd.config=/var/lib/cilium/etcd-config.yaml + {{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + - --max-connected-clusters={{ .Values.clustermesh.maxConnectedClusters }} + {{- end }} {{- if ne .Values.clustermesh.apiserver.tls.authMode "legacy" }} - --cluster-users-enabled - --cluster-users-config-path=/var/lib/cilium/etcd-config/users.yaml @@ -167,6 +176,7 @@ spec: - --enable-external-workloads={{ .Values.externalWorkloads.enabled }} {{- if .Values.clustermesh.apiserver.metrics.enabled }} - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.port }} + - --controller-group-metrics=all {{- end }} {{- with .Values.clustermesh.apiserver.extraArgs }} {{- toYaml . | trim | nindent 8 }} @@ -224,13 +234,18 @@ spec: securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.clustermesh.apiserver.lifecycle }} + lifecycle: + {{- toYaml . | nindent 10 }} + {{- end }} {{- if .Values.clustermesh.apiserver.kvstoremesh.enabled }} - name: kvstoremesh - image: {{ include "cilium.image" .Values.clustermesh.apiserver.kvstoremesh.image | quote }} - imagePullPolicy: {{ .Values.clustermesh.apiserver.kvstoremesh.image.pullPolicy }} + image: {{ include "cilium.image" (list $ .Values.clustermesh.apiserver.image) | quote }} + imagePullPolicy: {{ .Values.clustermesh.apiserver.image.pullPolicy }} command: - - /usr/bin/kvstoremesh + - /usr/bin/clustermesh-apiserver args: + - kvstoremesh {{- if .Values.debug.enabled }} - --debug {{- end }} @@ -240,8 +255,12 @@ spec: - --kvstore-opt=etcd.qps=100 - --kvstore-opt=etcd.maxInflight=10 - --clustermesh-config=/var/lib/cilium/clustermesh + {{- if hasKey .Values.clustermesh "maxConnectedClusters" }} + - --max-connected-clusters={{ .Values.clustermesh.maxConnectedClusters }} + {{- end }} {{- if .Values.clustermesh.apiserver.metrics.kvstoremesh.enabled }} - --prometheus-serve-addr=:{{ .Values.clustermesh.apiserver.metrics.kvstoremesh.port }} + - --controller-group-metrics=all {{- end }} {{- with .Values.clustermesh.apiserver.kvstoremesh.extraArgs }} {{- toYaml . | trim | nindent 8 }} @@ -285,6 +304,10 @@ spec: securityContext: {{- toYaml . | nindent 10 }} {{- end }} + {{- with .Values.clustermesh.apiserver.kvstoremesh.lifecycle }} + lifecycle: + {{- toYaml . | nindent 10 }} + {{- end }} {{- end }} volumes: - name: etcd-server-secrets @@ -371,6 +394,7 @@ spec: priorityClassName: {{ include "cilium.priorityClass" (list $ .Values.clustermesh.apiserver.priorityClassName "system-cluster-critical") }} serviceAccount: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} serviceAccountName: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} + terminationGracePeriodSeconds: {{ .Values.clustermesh.apiserver.terminationGracePeriodSeconds }} automountServiceAccountToken: {{ .Values.serviceAccounts.clustermeshApiserver.automount }} {{- with .Values.clustermesh.apiserver.affinity }} affinity: diff --git a/helm/cilium/templates/clustermesh-apiserver/metrics-service.yaml b/helm/cilium/templates/clustermesh-apiserver/metrics-service.yaml index 63b50ace..0781af1e 100644 --- a/helm/cilium/templates/clustermesh-apiserver/metrics-service.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/metrics-service.yaml @@ -7,6 +7,10 @@ kind: Service metadata: name: clustermesh-apiserver-metrics namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: clustermesh-apiserver app.kubernetes.io/part-of: cilium diff --git a/helm/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml b/helm/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml index 243e1b23..4a1bbf7e 100644 --- a/helm/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/poddisruptionbudget.yaml @@ -5,6 +5,10 @@ kind: PodDisruptionBudget metadata: name: clustermesh-apiserver namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: clustermesh-apiserver app.kubernetes.io/part-of: cilium diff --git a/helm/cilium/templates/clustermesh-apiserver/service.yaml b/helm/cilium/templates/clustermesh-apiserver/service.yaml index 466841d9..0a7028c5 100644 --- a/helm/cilium/templates/clustermesh-apiserver/service.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/service.yaml @@ -8,9 +8,14 @@ metadata: k8s-app: clustermesh-apiserver app.kubernetes.io/part-of: cilium app.kubernetes.io/name: clustermesh-apiserver - {{- with .Values.clustermesh.apiserver.service.annotations }} + {{- if or .Values.clustermesh.apiserver.service.annotations .Values.clustermesh.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.apiserver.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} spec: type: {{ .Values.clustermesh.apiserver.service.type }} diff --git a/helm/cilium/templates/clustermesh-apiserver/serviceaccount.yaml b/helm/cilium/templates/clustermesh-apiserver/serviceaccount.yaml index b6153c39..0c8992ba 100644 --- a/helm/cilium/templates/clustermesh-apiserver/serviceaccount.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.clustermeshApiserver.name | quote }} namespace: {{ .Release.Namespace }} - {{- with .Values.serviceAccounts.clustermeshApiserver.annotations }} + {{- if or .Values.serviceAccounts.clustermeshApiserver.annotations .Values.clustermesh.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.clustermeshApiserver.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/servicemonitor.yaml b/helm/cilium/templates/clustermesh-apiserver/servicemonitor.yaml index d5ba6420..8ecf150b 100644 --- a/helm/cilium/templates/clustermesh-apiserver/servicemonitor.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/servicemonitor.yaml @@ -14,10 +14,15 @@ metadata: {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if or .Values.clustermesh.apiserver.metrics.serviceMonitor.annotations .Values.clustermesh.annotations }} annotations: + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.clustermesh.apiserver.metrics.serviceMonitor.annotations }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: selector: matchLabels: diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml index 3bc84ae1..097cb14e 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/admin-secret.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: clustermesh-apiserver-admin-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml index 3c2cf643..f290fe8e 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/client-secret.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: clustermesh-apiserver-client-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml index 0a5e42e2..06bb8bd2 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/remote-secret.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: clustermesh-apiserver-remote-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml index 86fe56f5..f7cfd7ad 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-certmanager/server-secret.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: clustermesh-apiserver-server-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl index d61ec718..52a99089 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl +++ b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/_job-spec.tpl @@ -15,7 +15,7 @@ spec: spec: containers: - name: certgen - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.certgen.image }}" + image: {{ include "cilium.image" .Values.certgen.image | quote }} imagePullPolicy: {{ .Values.certgen.image.pullPolicy }} command: - "/usr/bin/cilium-certgen" @@ -26,12 +26,8 @@ spec: {{- end }} - "--ca-generate" - "--ca-reuse-secret" - {{- if .Values.clustermesh.apiserver.tls.ca.cert }} - - "--ca-secret-name=clustermesh-apiserver-ca-cert" - {{- else -}} - {{- if and .Values.tls.ca.cert .Values.tls.ca.key }} + {{- if and .Values.tls.ca.cert .Values.tls.ca.key }} - "--ca-secret-name=cilium-ca" - {{- end }} {{- end }} - "--clustermesh-apiserver-server-cert-generate" - "--clustermesh-apiserver-server-cert-validity-duration={{ $certValiditySecondsStr }}" @@ -69,5 +65,9 @@ spec: volumes: {{- toYaml . | nindent 6 }} {{- end }} + affinity: + {{- with .Values.certgen.affinity }} + {{- toYaml . | nindent 8 }} + {{- end }} ttlSecondsAfterFinished: {{ .Values.certgen.ttlSecondsAfterFinished }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/ca-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/ca-secret.yaml deleted file mode 100644 index 6f499cd8..00000000 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/ca-secret.yaml +++ /dev/null @@ -1,15 +0,0 @@ -{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "cronJob") }} -{{- $crt := .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert -}} -{{- $key := .Values.clustermesh.apiserver.tls.ca.key | default .Values.tls.ca.key -}} -{{- if and $crt $key }} ---- -apiVersion: v1 -kind: Secret -metadata: - name: clustermesh-apiserver-ca-cert - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ $crt }} - ca.key: {{ $key }} -{{- end }} -{{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml index 7711bc93..946602b4 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/cronjob.yaml @@ -4,6 +4,10 @@ kind: CronJob metadata: name: clustermesh-apiserver-generate-certs namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: clustermesh-apiserver-generate-certs app.kubernetes.io/part-of: cilium diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml index 06235f29..b6dd5bf1 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/job.yaml @@ -13,5 +13,8 @@ metadata: {{- with .Values.certgen.annotations.job }} {{- toYaml . | nindent 4 }} {{- end }} + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{ include "clustermesh-apiserver-generate-certs.job.spec" . }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml index 0d0565a6..79960cba 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/role.yaml @@ -4,6 +4,10 @@ kind: Role metadata: name: clustermesh-apiserver-generate-certs namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: @@ -19,7 +23,6 @@ rules: - secrets resourceNames: - cilium-ca - - clustermesh-apiserver-ca-cert verbs: - get - update diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml index 22fc3ed7..4e67e047 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/rolebinding.yaml @@ -4,6 +4,10 @@ kind: RoleBinding metadata: name: clustermesh-apiserver-generate-certs namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml index a6f79a66..8dfaf52c 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-cronjob/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.clustermeshcertgen.name | quote }} namespace: {{ .Release.Namespace }} - {{- with .Values.serviceAccounts.clustermeshcertgen.annotations }} + {{- if or .Values.serviceAccounts.clustermeshcertgen.annotations .Values.clustermesh.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.serviceAccounts.clustermeshcertgen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.clustermesh.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-helm/_helpers.tpl b/helm/cilium/templates/clustermesh-apiserver/tls-helm/_helpers.tpl deleted file mode 100644 index 576160f9..00000000 --- a/helm/cilium/templates/clustermesh-apiserver/tls-helm/_helpers.tpl +++ /dev/null @@ -1,37 +0,0 @@ -{{/* -Generate TLS certificates for ClusterMesh. - -Note: Always use this template as follows: - - {{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} - -The assignment to `$_` is required because we store the generated CI in a global `cmca` variable. -Please, don't try to "simplify" this, as without this trick, every generated -certificate would be signed by a different CA. -*/}} -{{- define "clustermesh-apiserver-generate-certs.helm.setup-ca" }} - {{- if not .cmca }} - {{- $ca := "" -}} - {{- $crt := .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert -}} - {{- $key := .Values.clustermesh.apiserver.tls.ca.key | default .Values.tls.ca.key -}} - {{- if and $crt $key }} - {{- $ca = buildCustomCert $crt $key -}} - {{- else }} - {{- with lookup "v1" "Secret" .Release.Namespace "clustermesh-apiserver-ca-cert" }} - {{- $crt := index .data "ca.crt" }} - {{- $key := index .data "ca.key" }} - {{- $ca = buildCustomCert $crt $key -}} - {{- else }} - {{- $_ := include "cilium.ca.setup" . -}} - {{- with lookup "v1" "Secret" .Release.Namespace .commonCASecretName }} - {{- $crt := index .data "ca.crt" }} - {{- $key := index .data "ca.key" }} - {{- $ca = buildCustomCert $crt $key -}} - {{- else }} - {{- $ca = .commonCA -}} - {{- end }} - {{- end }} - {{- end }} - {{- $_ := set . "cmca" $ca -}} - {{- end }} -{{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml index dfa1d987..63282c98 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-helm/admin-secret.yaml @@ -1,17 +1,21 @@ {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} -{{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} +{{- $_ := include "cilium.ca.setup" . -}} {{- $cn := include "clustermesh-apiserver-generate-certs.admin-common-name" . -}} {{- $dns := list "localhost" }} -{{- $cert := genSignedCert $cn nil $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} +{{- $cert := genSignedCert $cn nil $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} --- apiVersion: v1 kind: Secret metadata: name: clustermesh-apiserver-admin-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .cmca.Cert | b64enc }} + ca.crt: {{ .commonCA.Cert | b64enc }} tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-helm/ca-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-helm/ca-secret.yaml deleted file mode 100644 index 832c6bb5..00000000 --- a/helm/cilium/templates/clustermesh-apiserver/tls-helm/ca-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} -{{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} ---- -apiVersion: v1 -kind: Secret -metadata: - name: clustermesh-apiserver-ca-cert - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ .cmca.Cert | b64enc }} - ca.key: {{ .cmca.Key | b64enc }} -{{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml index fd943340..85137111 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-helm/client-secret.yaml @@ -1,16 +1,20 @@ {{- if and .Values.externalWorkloads.enabled .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} -{{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} +{{- $_ := include "cilium.ca.setup" . -}} {{- $cn := "externalworkload" }} -{{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} +{{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} --- apiVersion: v1 kind: Secret metadata: name: clustermesh-apiserver-client-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .cmca.Cert | b64enc }} + ca.crt: {{ .commonCA.Cert | b64enc }} tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml index 4a68c747..7bfb1c5a 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-helm/remote-secret.yaml @@ -1,16 +1,20 @@ {{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} -{{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} +{{- $_ := include "cilium.ca.setup" . -}} {{- $cn := include "clustermesh-apiserver-generate-certs.remote-common-name" . -}} -{{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} +{{- $cert := genSignedCert $cn nil nil (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} --- apiVersion: v1 kind: Secret metadata: name: clustermesh-apiserver-remote-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .cmca.Cert | b64enc }} + ca.crt: {{ .commonCA.Cert | b64enc }} tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml index beacb063..f958f353 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-helm/server-secret.yaml @@ -1,18 +1,22 @@ {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "helm") }} -{{- $_ := include "clustermesh-apiserver-generate-certs.helm.setup-ca" . -}} +{{- $_ := include "cilium.ca.setup" . -}} {{- $cn := "clustermesh-apiserver.cilium.io" }} {{- $ip := concat (list "127.0.0.1" "::1") .Values.clustermesh.apiserver.tls.server.extraIpAddresses }} {{- $dns := concat (list $cn "*.mesh.cilium.io" (printf "clustermesh-apiserver.%s.svc" .Release.Namespace)) .Values.clustermesh.apiserver.tls.server.extraDnsNames }} -{{- $cert := genSignedCert $cn $ip $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .cmca -}} +{{- $cert := genSignedCert $cn $ip $dns (.Values.clustermesh.apiserver.tls.auto.certValidityDuration | int) .commonCA -}} --- apiVersion: v1 kind: Secret metadata: name: clustermesh-apiserver-server-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .cmca.Cert | b64enc }} + ca.crt: {{ .commonCA.Cert | b64enc }} tls.crt: {{ $cert.Cert | b64enc }} tls.key: {{ $cert.Key | b64enc }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml index 4ff56ab2..de038ca8 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-provided/admin-secret.yaml @@ -4,9 +4,13 @@ kind: Secret metadata: name: clustermesh-apiserver-admin-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.admin.cert | required "missing clustermesh.apiserver.tls.admin.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.admin.key | required "missing clustermesh.apiserver.tls.admin.key" }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml deleted file mode 100644 index 9b29a541..00000000 --- a/helm/cilium/templates/clustermesh-apiserver/tls-provided/ca-secret.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) (not .Values.clustermesh.apiserver.tls.auto.enabled) }} -apiVersion: v1 -kind: Secret -metadata: - name: clustermesh-apiserver-ca-cert - namespace: {{ .Release.Namespace }} -data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} - {{- if .Values.clustermesh.apiserver.tls.ca.key | default .Values.tls.ca.key }} - ca.key: {{ .Values.clustermesh.apiserver.tls.ca.key | default .Values.tls.ca.key }} - {{- end }} -{{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml index 02a180ff..b8ea7b60 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-provided/client-secret.yaml @@ -4,9 +4,13 @@ kind: Secret metadata: name: clustermesh-apiserver-client-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.client.cert | required "missing clustermesh.apiserver.tls.client.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.client.key | required "missing clustermesh.apiserver.tls.client.key" }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml index 7850b332..8a253eb5 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-provided/remote-secret.yaml @@ -4,9 +4,13 @@ kind: Secret metadata: name: clustermesh-apiserver-remote-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.remote.cert | required "missing clustermesh.apiserver.tls.remote.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.remote.key | required "missing clustermesh.apiserver.tls.remote.key" }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml b/helm/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml index bb160d62..8c239b07 100644 --- a/helm/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/tls-provided/server-secret.yaml @@ -4,9 +4,13 @@ kind: Secret metadata: name: clustermesh-apiserver-server-cert namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: - ca.crt: {{ .Values.clustermesh.apiserver.tls.ca.cert | default .Values.tls.ca.cert }} + ca.crt: {{ .Values.tls.ca.cert }} tls.crt: {{ .Values.clustermesh.apiserver.tls.server.cert | required "missing clustermesh.apiserver.tls.server.cert" }} tls.key: {{ .Values.clustermesh.apiserver.tls.server.key | required "missing clustermesh.apiserver.tls.server.key" }} {{- end }} diff --git a/helm/cilium/templates/clustermesh-apiserver/users-configmap.yaml b/helm/cilium/templates/clustermesh-apiserver/users-configmap.yaml index 7c278978..e1cbf95f 100644 --- a/helm/cilium/templates/clustermesh-apiserver/users-configmap.yaml +++ b/helm/cilium/templates/clustermesh-apiserver/users-configmap.yaml @@ -5,6 +5,10 @@ kind: ConfigMap metadata: name: clustermesh-remote-users namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium data: diff --git a/helm/cilium/templates/clustermesh-config/_helpers.tpl b/helm/cilium/templates/clustermesh-config/_helpers.tpl index f492275f..5cd31486 100644 --- a/helm/cilium/templates/clustermesh-config/_helpers.tpl +++ b/helm/cilium/templates/clustermesh-config/_helpers.tpl @@ -1,8 +1,7 @@ {{- define "clustermesh-config-generate-etcd-cfg" }} {{- $cluster := index . 0 -}} {{- $domain := index . 1 -}} -{{- $hasCustomCACert := index . 2 -}} -{{- $override := index . 3 -}} +{{- $override := index . 2 -}} {{- /* The parenthesis around $cluster.tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} {{- $prefix := ternary "common-" (printf "%s." $cluster.name) (or (ne $override "") (empty ($cluster.tls).cert) (empty ($cluster.tls).key)) -}} @@ -14,7 +13,7 @@ endpoints: {{- else }} - https://{{ $cluster.address | required "missing clustermesh.apiserver.config.clusters.address" }}:{{ $cluster.port }} {{- end }} -{{- if $hasCustomCACert }} +{{- if not (empty ($cluster.tls).caCert) }} {{- /* The custom CA configuration takes effect only if a custom certificate and key are also set */}} trusted-ca-file: /var/lib/cilium/clustermesh/{{ $prefix }}etcd-client-ca.crt {{- else }} diff --git a/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml b/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml index 36e23b4e..78ddecd9 100644 --- a/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml +++ b/helm/cilium/templates/clustermesh-config/clustermesh-secret.yaml @@ -1,19 +1,23 @@ -{{- if and .Values.clustermesh.useAPIServer .Values.clustermesh.config.enabled }} +{{- if .Values.clustermesh.config.enabled }} --- apiVersion: v1 kind: Secret metadata: name: cilium-clustermesh namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: - {{- $override := ternary (printf "https://clustermesh-apiserver.%s.svc:2379" .Release.Namespace) "" $.Values.clustermesh.apiserver.kvstoremesh.enabled }} + {{- $kvstoremesh := and .Values.clustermesh.useAPIServer .Values.clustermesh.apiserver.kvstoremesh.enabled }} + {{- $override := ternary (printf "https://clustermesh-apiserver.%s.svc:2379" .Release.Namespace) "" $kvstoremesh }} {{- range .Values.clustermesh.config.clusters }} - {{- $hasCustomCACert := or (.tls).caCert $.Values.clustermesh.apiserver.tls.ca.cert }} - {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert $override) | b64enc }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $override) | b64enc }} {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} {{- if and (eq $override "") (.tls).cert (.tls).key }} - {{- if $hasCustomCACert }} - {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert | default $.Values.clustermesh.apiserver.tls.ca.cert }} + {{- if .tls.caCert }} + {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert }} {{- end }} {{ .name }}.etcd-client.key: {{ .tls.key }} {{ .name }}.etcd-client.crt: {{ .tls.cert }} diff --git a/helm/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml b/helm/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml index f79c4ade..91ae0320 100644 --- a/helm/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml +++ b/helm/cilium/templates/clustermesh-config/kvstoremesh-secret.yaml @@ -5,14 +5,17 @@ kind: Secret metadata: name: cilium-kvstoremesh namespace: {{ .Release.Namespace }} + {{- with .Values.clustermesh.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: {{- range .Values.clustermesh.config.clusters }} - {{- $hasCustomCACert := or (.tls).caCert $.Values.clustermesh.apiserver.tls.ca.cert }} - {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain $hasCustomCACert "") | b64enc }} + {{ .name }}: {{ include "clustermesh-config-generate-etcd-cfg" (list . $.Values.clustermesh.config.domain "") | b64enc }} {{- /* The parenthesis around .tls are required, since it can be null: https://stackoverflow.com/a/68807258 */}} {{- if and (.tls).cert (.tls).key }} - {{- if $hasCustomCACert }} - {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert | default $.Values.clustermesh.apiserver.tls.ca.cert }} + {{- if .tls.caCert }} + {{ .name }}.etcd-client-ca.crt: {{ .tls.caCert }} {{- end }} {{ .name }}.etcd-client.key: {{ .tls.key }} {{ .name }}.etcd-client.crt: {{ .tls.cert }} diff --git a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml index 2f0b6c22..22a81c2c 100644 --- a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml +++ b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrole.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: cilium-etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: diff --git a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml index e9578f7f..25a92e10 100644 --- a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml +++ b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-clusterrolebinding.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: cilium-etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml index aad6d4f2..e5b730b5 100644 --- a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml +++ b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-deployment.yaml @@ -9,6 +9,10 @@ metadata: app.kubernetes.io/part-of: cilium name: cilium-etcd-operator namespace: {{ .Release.Namespace }} + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: replicas: 1 selector: @@ -90,7 +94,7 @@ spec: value: "revision" - name: CILIUM_ETCD_META_ETCD_AUTO_COMPACTION_RETENTION value: "25000" - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.etcd.image }}" + image: {{ include "cilium.image" (list $ .Values.etcd.image) | quote }} imagePullPolicy: {{ .Values.etcd.image.pullPolicy }} name: cilium-etcd-operator terminationMessagePolicy: FallbackToLogsOnError diff --git a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml index 9bc0a3ea..7d7b765a 100644 --- a/helm/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml +++ b/helm/cilium/templates/etcd-operator/cilium-etcd-operator-serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.etcd.name | quote }} namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.etcd.annotations }} + {{- if or .Values.serviceAccounts.etcd.annotations .Values.etcd.annotations }} annotations: -{{ toYaml .Values.serviceAccounts.etcd.annotations | indent 4 }} + {{- with .Values.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml b/helm/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml index 424498f6..72cb4e6e 100644 --- a/helm/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml +++ b/helm/cilium/templates/etcd-operator/etcd-operator-clusterrole.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: diff --git a/helm/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml b/helm/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml index a56624d0..b729ea71 100644 --- a/helm/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml +++ b/helm/cilium/templates/etcd-operator/etcd-operator-clusterrolebinding.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: etcd-operator + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml b/helm/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml index 278d9842..06ee39c1 100644 --- a/helm/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml +++ b/helm/cilium/templates/etcd-operator/etcd-operator-serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: cilium-etcd-sa namespace: {{ .Release.Namespace }} - {{- if .Values.serviceAccounts.etcd.annotations }} + {{- if or .Values.serviceAccounts.etcd.annotations .Values.etcd.annotations }} annotations: -{{ toYaml .Values.serviceAccounts.etcd.annotations | indent 4 }} + {{- with .Values.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.etcd.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/etcd-operator/poddisruptionbudget.yaml b/helm/cilium/templates/etcd-operator/poddisruptionbudget.yaml index 72e0b835..5939b4ae 100644 --- a/helm/cilium/templates/etcd-operator/poddisruptionbudget.yaml +++ b/helm/cilium/templates/etcd-operator/poddisruptionbudget.yaml @@ -5,6 +5,10 @@ kind: PodDisruptionBudget metadata: name: cilium-etcd-operator namespace: {{ .Release.Namespace }} + {{- with .Values.etcd.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: io.cilium/app: etcd-operator name: cilium-etcd-operator diff --git a/helm/cilium/templates/hubble-relay/configmap.yaml b/helm/cilium/templates/hubble-relay/configmap.yaml index 0f5a037d..93f5b8d8 100644 --- a/helm/cilium/templates/hubble-relay/configmap.yaml +++ b/helm/cilium/templates/hubble-relay/configmap.yaml @@ -9,6 +9,10 @@ kind: ConfigMap metadata: name: hubble-relay-config namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: config.yaml: | cluster-name: {{ .Values.cluster.name }} diff --git a/helm/cilium/templates/hubble-relay/deployment.yaml b/helm/cilium/templates/hubble-relay/deployment.yaml index c6831258..525885cd 100644 --- a/helm/cilium/templates/hubble-relay/deployment.yaml +++ b/helm/cilium/templates/hubble-relay/deployment.yaml @@ -4,6 +4,10 @@ kind: Deployment metadata: name: hubble-relay namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: hubble-relay app.kubernetes.io/name: hubble-relay @@ -49,7 +53,7 @@ spec: securityContext: {{- toYaml . | nindent 12 }} {{- end }} - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.hubble.relay.image }}" + image: {{ include "cilium.image" (list $ .Values.hubble.relay.image) | quote }} imagePullPolicy: {{ .Values.hubble.relay.image.pullPolicy }} command: - hubble-relay @@ -67,11 +71,26 @@ spec: protocol: TCP {{- end }} readinessProbe: - tcpSocket: - port: grpc + {{- include "hubble-relay.probe" . | nindent 12 }} + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 5 + {{- end }} livenessProbe: - tcpSocket: - port: grpc + {{- include "hubble-relay.probe" . | nindent 12 }} + {{- if semverCompare "<1.20-0" .Capabilities.KubeVersion.Version }} + # Starting from Kubernetes 1.20, we are using startupProbe instead + # of this field. + initialDelaySeconds: 60 + {{- end }} + {{- if semverCompare ">=1.20-0" .Capabilities.KubeVersion.Version }} + startupProbe: + # give the relay one minute to start up + {{- include "hubble-relay.probe" . | nindent 12 }} + failureThreshold: 20 + periodSeconds: 3 + {{- end }} {{- with .Values.hubble.relay.extraEnv }} env: {{- toYaml . | trim | nindent 12 }} @@ -166,3 +185,17 @@ spec: {{- toYaml . | nindent 6 }} {{- end }} {{- end }} + +{{- define "hubble-relay.probe" }} +{{- /* This distinction can be removed once we drop support for k8s 1.23 */}} +{{- if semverCompare ">=1.24-0" .Capabilities.KubeVersion.Version -}} +grpc: + port: 4222 +{{- else }} +exec: + command: + - grpc_health_probe + - -addr=localhost:4222 +{{- end }} +timeoutSeconds: 3 +{{- end }} diff --git a/helm/cilium/templates/hubble-relay/metrics-service.yaml b/helm/cilium/templates/hubble-relay/metrics-service.yaml index 5b7c99c9..1066c6c4 100644 --- a/helm/cilium/templates/hubble-relay/metrics-service.yaml +++ b/helm/cilium/templates/hubble-relay/metrics-service.yaml @@ -5,6 +5,10 @@ apiVersion: v1 metadata: name: hubble-relay-metrics namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: hubble-relay spec: diff --git a/helm/cilium/templates/hubble-relay/poddisruptionbudget.yaml b/helm/cilium/templates/hubble-relay/poddisruptionbudget.yaml index 8a90916a..4fd6da9b 100644 --- a/helm/cilium/templates/hubble-relay/poddisruptionbudget.yaml +++ b/helm/cilium/templates/hubble-relay/poddisruptionbudget.yaml @@ -5,6 +5,10 @@ kind: PodDisruptionBudget metadata: name: hubble-relay namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: hubble-relay app.kubernetes.io/name: hubble-relay diff --git a/helm/cilium/templates/hubble-relay/service.yaml b/helm/cilium/templates/hubble-relay/service.yaml index d995c419..39259473 100644 --- a/helm/cilium/templates/hubble-relay/service.yaml +++ b/helm/cilium/templates/hubble-relay/service.yaml @@ -4,6 +4,10 @@ apiVersion: v1 metadata: name: hubble-relay namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.relay.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: hubble-relay app.kubernetes.io/name: hubble-relay diff --git a/helm/cilium/templates/hubble-relay/serviceaccount.yaml b/helm/cilium/templates/hubble-relay/serviceaccount.yaml index f42bd1b0..cf56d631 100644 --- a/helm/cilium/templates/hubble-relay/serviceaccount.yaml +++ b/helm/cilium/templates/hubble-relay/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.relay.name | quote }} namespace: {{ .Release.Namespace }} - {{- with .Values.serviceAccounts.relay.annotations }} + {{- if or .Values.serviceAccounts.relay.annotations .Values.hubble.relay.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.hubble.relay.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.relay.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/hubble-relay/servicemonitor.yaml b/helm/cilium/templates/hubble-relay/servicemonitor.yaml index 0b9f1622..4e41fdf3 100644 --- a/helm/cilium/templates/hubble-relay/servicemonitor.yaml +++ b/helm/cilium/templates/hubble-relay/servicemonitor.yaml @@ -8,10 +8,15 @@ metadata: {{- with .Values.hubble.relay.prometheus.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if or .Values.hubble.relay.prometheus.serviceMonitor.annotations .Values.hubble.relay.annotations }} annotations: + {{- with .Values.hubble.relay.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.hubble.relay.prometheus.serviceMonitor.annotations }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: selector: matchLabels: diff --git a/helm/cilium/templates/hubble-ui/_nginx.tpl b/helm/cilium/templates/hubble-ui/_nginx.tpl index 3b409207..e787b5aa 100644 --- a/helm/cilium/templates/hubble-ui/_nginx.tpl +++ b/helm/cilium/templates/hubble-ui/_nginx.tpl @@ -49,6 +49,13 @@ server { # double `/index.html` is required here try_files $uri $uri/ /index.html /index.html; } + + # Liveness probe + location /healthz { + access_log off; + add_header Content-Type text/plain; + return 200 'ok'; + } } } {{- end }} diff --git a/helm/cilium/templates/hubble-ui/clusterrole.yaml b/helm/cilium/templates/hubble-ui/clusterrole.yaml index c3fa7809..5df709f7 100644 --- a/helm/cilium/templates/hubble-ui/clusterrole.yaml +++ b/helm/cilium/templates/hubble-ui/clusterrole.yaml @@ -3,6 +3,10 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: hubble-ui + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: diff --git a/helm/cilium/templates/hubble-ui/clusterrolebinding.yaml b/helm/cilium/templates/hubble-ui/clusterrolebinding.yaml index 16c688df..d091786b 100644 --- a/helm/cilium/templates/hubble-ui/clusterrolebinding.yaml +++ b/helm/cilium/templates/hubble-ui/clusterrolebinding.yaml @@ -3,6 +3,10 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: hubble-ui + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/hubble-ui/configmap.yaml b/helm/cilium/templates/hubble-ui/configmap.yaml index bbab253d..8b5f0141 100644 --- a/helm/cilium/templates/hubble-ui/configmap.yaml +++ b/helm/cilium/templates/hubble-ui/configmap.yaml @@ -5,6 +5,10 @@ kind: ConfigMap metadata: name: hubble-ui-nginx namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: nginx.conf: {{ include "hubble-ui.nginx.conf" . | trim | quote }} {{- end }} diff --git a/helm/cilium/templates/hubble-ui/deployment.yaml b/helm/cilium/templates/hubble-ui/deployment.yaml index dc4306f7..0ef721dd 100644 --- a/helm/cilium/templates/hubble-ui/deployment.yaml +++ b/helm/cilium/templates/hubble-ui/deployment.yaml @@ -4,6 +4,10 @@ apiVersion: apps/v1 metadata: name: hubble-ui namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: hubble-ui app.kubernetes.io/name: hubble-ui @@ -51,7 +55,7 @@ spec: {{- end }} containers: - name: frontend - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.hubble.ui.frontend.image }}" + image: {{ include "cilium.image" (list $ .Values.hubble.ui.frontend.image) | quote }} imagePullPolicy: {{ .Values.hubble.ui.frontend.image.pullPolicy }} ports: - name: http @@ -60,6 +64,14 @@ spec: env: {{- toYaml . | trim | nindent 12 }} {{- end }} + livenessProbe: + httpGet: + path: /healthz + port: 8081 + readinessProbe: + httpGet: + path: / + port: 8081 {{- with .Values.hubble.ui.frontend.resources }} resources: {{- toYaml . | trim | nindent 10 }} @@ -79,7 +91,7 @@ spec: {{- toYaml . | trim | nindent 10 }} {{- end }} - name: backend - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.hubble.ui.backend.image }}" + image: {{ include "cilium.image" (list $ .Values.hubble.ui.backend.image) | quote }} imagePullPolicy: {{ .Values.hubble.ui.backend.image.pullPolicy }} env: - name: EVENTS_SERVER_PORT @@ -90,7 +102,7 @@ spec: - name: TLS_TO_RELAY_ENABLED value: "true" - name: TLS_RELAY_SERVER_NAME - value: ui.hubble-relay.cilium.io + value: {{ .Values.hubble.relay.tls.server.relayName }} - name: TLS_RELAY_CA_CERT_FILES value: /var/lib/hubble-ui/certs/hubble-relay-ca.crt - name: TLS_RELAY_CLIENT_CERT_FILE @@ -104,6 +116,18 @@ spec: {{- with .Values.hubble.ui.backend.extraEnv }} {{- toYaml . | trim | nindent 8 }} {{- end }} + {{- if .Values.hubble.ui.backend.livenessProbe.enabled }} + livenessProbe: + httpGet: + path: /healthz + port: 8090 + {{- end }} + {{- if .Values.hubble.ui.backend.readinessProbe.enabled }} + readinessProbe: + httpGet: + path: /healthz + port: 8090 + {{- end }} ports: - name: grpc containerPort: 8090 diff --git a/helm/cilium/templates/hubble-ui/ingress.yaml b/helm/cilium/templates/hubble-ui/ingress.yaml index b48e2cb4..2c0ff7d3 100644 --- a/helm/cilium/templates/hubble-ui/ingress.yaml +++ b/helm/cilium/templates/hubble-ui/ingress.yaml @@ -12,9 +12,14 @@ metadata: {{- with .Values.hubble.ui.ingress.labels }} {{- toYaml . | nindent 4 }} {{- end }} - {{- with .Values.hubble.ui.ingress.annotations }} + {{- if or .Values.hubble.ui.ingress.annotations .Values.hubble.ui.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.hubble.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.ui.ingress.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} spec: {{- if .Values.hubble.ui.ingress.className }} diff --git a/helm/cilium/templates/hubble-ui/poddisruptionbudget.yaml b/helm/cilium/templates/hubble-ui/poddisruptionbudget.yaml index 3108a493..af3b6705 100644 --- a/helm/cilium/templates/hubble-ui/poddisruptionbudget.yaml +++ b/helm/cilium/templates/hubble-ui/poddisruptionbudget.yaml @@ -5,6 +5,10 @@ kind: PodDisruptionBudget metadata: name: hubble-ui namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.ui.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: hubble-ui app.kubernetes.io/name: hubble-ui diff --git a/helm/cilium/templates/hubble-ui/service.yaml b/helm/cilium/templates/hubble-ui/service.yaml index 52eb6eec..a820b342 100644 --- a/helm/cilium/templates/hubble-ui/service.yaml +++ b/helm/cilium/templates/hubble-ui/service.yaml @@ -4,9 +4,14 @@ apiVersion: v1 metadata: name: hubble-ui namespace: {{ .Release.Namespace }} - {{- if .Values.hubble.ui.service.annotations }} + {{- if or .Values.hubble.ui.service.annotations .Values.hubble.ui.annotations }} annotations: - {{- toYaml .Values.hubble.ui.service.annotations | nindent 4 }} + {{- with .Values.hubble.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.hubble.ui.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} labels: k8s-app: hubble-ui diff --git a/helm/cilium/templates/hubble-ui/serviceaccount.yaml b/helm/cilium/templates/hubble-ui/serviceaccount.yaml index 28f6061c..dc02ea2b 100644 --- a/helm/cilium/templates/hubble-ui/serviceaccount.yaml +++ b/helm/cilium/templates/hubble-ui/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.ui.name | quote }} namespace: {{ .Release.Namespace }} - {{- with .Values.serviceAccounts.ui.annotations }} + {{- if or .Values.serviceAccounts.ui.annotations .Values.hubble.ui.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.hubble.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.ui.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/hubble/dashboards-configmap.yaml b/helm/cilium/templates/hubble/dashboards-configmap.yaml index c8453560..c668ebfd 100644 --- a/helm/cilium/templates/hubble/dashboards-configmap.yaml +++ b/helm/cilium/templates/hubble/dashboards-configmap.yaml @@ -15,9 +15,14 @@ metadata: {{- if $.Values.hubble.metrics.dashboards.label }} {{ $.Values.hubble.metrics.dashboards.label }}: {{ ternary $.Values.hubble.metrics.dashboards.labelValue "1" (not (empty $.Values.hubble.metrics.dashboards.labelValue)) | quote }} {{- end }} - {{- with $.Values.hubble.metrics.dashboards.annotations }} + {{- if or $.Values.hubble.metrics.dashboards.annotations $.Values.hubble.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with $.Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with $.Values.hubble.metrics.dashboards.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} data: {{ $dashboardName }}.json: {{ $.Files.Get $path | toJson }} diff --git a/helm/cilium/templates/hubble/metrics-service.yaml b/helm/cilium/templates/hubble/metrics-service.yaml index 0c72c0de..ab5cfbab 100644 --- a/helm/cilium/templates/hubble/metrics-service.yaml +++ b/helm/cilium/templates/hubble/metrics-service.yaml @@ -9,6 +9,9 @@ metadata: app.kubernetes.io/name: hubble app.kubernetes.io/part-of: cilium annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.hubble.metrics.serviceAnnotations }} {{- toYaml . | nindent 4 }} {{- end }} diff --git a/helm/cilium/templates/hubble/peer-service.yaml b/helm/cilium/templates/hubble/peer-service.yaml index f54fa6c6..7ba56456 100644 --- a/helm/cilium/templates/hubble/peer-service.yaml +++ b/helm/cilium/templates/hubble/peer-service.yaml @@ -4,6 +4,10 @@ kind: Service metadata: name: hubble-peer namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: k8s-app: cilium app.kubernetes.io/part-of: cilium diff --git a/helm/cilium/templates/hubble/servicemonitor.yaml b/helm/cilium/templates/hubble/servicemonitor.yaml index 27304ee8..3b3ba8ba 100644 --- a/helm/cilium/templates/hubble/servicemonitor.yaml +++ b/helm/cilium/templates/hubble/servicemonitor.yaml @@ -9,10 +9,15 @@ metadata: {{- with .Values.hubble.metrics.serviceMonitor.labels }} {{- toYaml . | nindent 4 }} {{- end }} + {{- if or .Values.hubble.metrics.serviceMonitor.annotations .Values.hubble.annotations }} annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.hubble.metrics.serviceMonitor.annotations }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: selector: matchLabels: @@ -33,4 +38,7 @@ spec: metricRelabelings: {{- toYaml . | nindent 4 }} {{- end }} +{{- if .Values.hubble.metrics.serviceMonitor.jobLabel }} + jobLabel: {{ .Values.hubble.metrics.serviceMonitor.jobLabel | quote }} +{{- end }} {{- end }} diff --git a/helm/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml b/helm/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml index 23bea64b..1dd96b18 100644 --- a/helm/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml +++ b/helm/cilium/templates/hubble/tls-certmanager/relay-client-secret.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: hubble-relay-client-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml b/helm/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml index 77f2800e..845b4fb8 100644 --- a/helm/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml +++ b/helm/cilium/templates/hubble/tls-certmanager/relay-server-secret.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: hubble-relay-server-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/hubble/tls-certmanager/server-secret.yaml b/helm/cilium/templates/hubble/tls-certmanager/server-secret.yaml index 1ca815fb..5f202e10 100644 --- a/helm/cilium/templates/hubble/tls-certmanager/server-secret.yaml +++ b/helm/cilium/templates/hubble/tls-certmanager/server-secret.yaml @@ -6,6 +6,10 @@ kind: Certificate metadata: name: hubble-server-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml b/helm/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml index 88d59490..5006666e 100644 --- a/helm/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml +++ b/helm/cilium/templates/hubble/tls-certmanager/ui-client-certs.yaml @@ -5,6 +5,10 @@ kind: Certificate metadata: name: hubble-ui-client-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} spec: issuerRef: {{- toYaml .Values.hubble.tls.auto.certManagerIssuerRef | nindent 4 }} diff --git a/helm/cilium/templates/hubble/tls-cronjob/_job-spec.tpl b/helm/cilium/templates/hubble/tls-cronjob/_job-spec.tpl index 3b3b6f31..4bee347a 100644 --- a/helm/cilium/templates/hubble/tls-cronjob/_job-spec.tpl +++ b/helm/cilium/templates/hubble/tls-cronjob/_job-spec.tpl @@ -11,7 +11,7 @@ spec: spec: containers: - name: certgen - image: "{{ .Values.image.registry }}/{{ include "cilium.image" .Values.certgen.image }}" + image: {{ include "cilium.image" .Values.certgen.image | quote }} imagePullPolicy: {{ .Values.certgen.image.pullPolicy }} command: - "/usr/bin/cilium-certgen" @@ -60,5 +60,9 @@ spec: volumes: {{- toYaml . | nindent 6 }} {{- end }} + affinity: + {{- with .Values.certgen.affinity }} + {{- toYaml . | nindent 8 }} + {{- end }} ttlSecondsAfterFinished: {{ .Values.certgen.ttlSecondsAfterFinished }} {{- end }} diff --git a/helm/cilium/templates/hubble/tls-cronjob/clusterrole.yaml b/helm/cilium/templates/hubble/tls-cronjob/clusterrole.yaml index dc075670..74d07831 100644 --- a/helm/cilium/templates/hubble/tls-cronjob/clusterrole.yaml +++ b/helm/cilium/templates/hubble/tls-cronjob/clusterrole.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: hubble-generate-certs + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium rules: diff --git a/helm/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml b/helm/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml index ebe60279..5938f16c 100644 --- a/helm/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml +++ b/helm/cilium/templates/hubble/tls-cronjob/clusterrolebinding.yaml @@ -3,6 +3,10 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: hubble-generate-certs + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} labels: app.kubernetes.io/part-of: cilium roleRef: diff --git a/helm/cilium/templates/hubble/tls-cronjob/cronjob.yaml b/helm/cilium/templates/hubble/tls-cronjob/cronjob.yaml index 5a5992cf..fa996608 100644 --- a/helm/cilium/templates/hubble/tls-cronjob/cronjob.yaml +++ b/helm/cilium/templates/hubble/tls-cronjob/cronjob.yaml @@ -8,10 +8,15 @@ metadata: k8s-app: hubble-generate-certs app.kubernetes.io/name: hubble-generate-certs app.kubernetes.io/part-of: cilium + {{- if or .Values.certgen.annotations.cronJob .Values.hubble.annotations }} annotations: + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- with .Values.certgen.annotations.cronJob }} - {{- toYaml . | nindent 4 }} + {{- toYaml . | nindent 4 }} {{- end }} + {{- end }} spec: schedule: {{ .Values.hubble.tls.auto.schedule | quote }} concurrencyPolicy: Forbid diff --git a/helm/cilium/templates/hubble/tls-cronjob/job.yaml b/helm/cilium/templates/hubble/tls-cronjob/job.yaml index e748165a..9831fdc6 100644 --- a/helm/cilium/templates/hubble/tls-cronjob/job.yaml +++ b/helm/cilium/templates/hubble/tls-cronjob/job.yaml @@ -14,5 +14,8 @@ metadata: {{- with .Values.certgen.annotations.job }} {{- toYaml . | nindent 4 }} {{- end }} + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{ include "hubble-generate-certs.job.spec" . }} {{- end }} diff --git a/helm/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml b/helm/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml index d538d67d..62a8de80 100644 --- a/helm/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml +++ b/helm/cilium/templates/hubble/tls-cronjob/serviceaccount.yaml @@ -4,8 +4,13 @@ kind: ServiceAccount metadata: name: {{ .Values.serviceAccounts.hubblecertgen.name | quote }} namespace: {{ .Release.Namespace }} - {{- with .Values.serviceAccounts.hubblecertgen.annotations }} + {{- if or .Values.serviceAccounts.hubblecertgen.annotations .Values.hubble.annotations }} annotations: - {{- toYaml . | nindent 4 }} + {{- with .Values.hubble.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.serviceAccounts.hubblecertgen.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- end }} diff --git a/helm/cilium/templates/hubble/tls-helm/relay-client-secret.yaml b/helm/cilium/templates/hubble/tls-helm/relay-client-secret.yaml index 947565ea..e1d6e876 100644 --- a/helm/cilium/templates/hubble/tls-helm/relay-client-secret.yaml +++ b/helm/cilium/templates/hubble/tls-helm/relay-client-secret.yaml @@ -9,6 +9,10 @@ kind: Secret metadata: name: hubble-relay-client-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .ca.Cert | b64enc }} diff --git a/helm/cilium/templates/hubble/tls-helm/relay-server-secret.yaml b/helm/cilium/templates/hubble/tls-helm/relay-server-secret.yaml index 2c2339d6..902c2be4 100644 --- a/helm/cilium/templates/hubble/tls-helm/relay-server-secret.yaml +++ b/helm/cilium/templates/hubble/tls-helm/relay-server-secret.yaml @@ -10,6 +10,10 @@ kind: Secret metadata: name: hubble-relay-server-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .ca.Cert | b64enc }} diff --git a/helm/cilium/templates/hubble/tls-helm/server-secret.yaml b/helm/cilium/templates/hubble/tls-helm/server-secret.yaml index cc3c1d91..a05c3266 100644 --- a/helm/cilium/templates/hubble/tls-helm/server-secret.yaml +++ b/helm/cilium/templates/hubble/tls-helm/server-secret.yaml @@ -10,6 +10,10 @@ kind: Secret metadata: name: hubble-server-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .ca.Cert | b64enc }} diff --git a/helm/cilium/templates/hubble/tls-helm/ui-client-certs.yaml b/helm/cilium/templates/hubble/tls-helm/ui-client-certs.yaml index 90376d65..7b385b26 100644 --- a/helm/cilium/templates/hubble/tls-helm/ui-client-certs.yaml +++ b/helm/cilium/templates/hubble/tls-helm/ui-client-certs.yaml @@ -9,6 +9,10 @@ kind: Secret metadata: name: hubble-ui-client-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .ca.Cert | b64enc }} diff --git a/helm/cilium/templates/hubble/tls-provided/relay-client-secret.yaml b/helm/cilium/templates/hubble/tls-provided/relay-client-secret.yaml index 4a15f646..b1512ed2 100644 --- a/helm/cilium/templates/hubble/tls-provided/relay-client-secret.yaml +++ b/helm/cilium/templates/hubble/tls-provided/relay-client-secret.yaml @@ -4,6 +4,10 @@ kind: Secret metadata: name: hubble-relay-client-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .Values.tls.ca.cert }} diff --git a/helm/cilium/templates/hubble/tls-provided/relay-server-secret.yaml b/helm/cilium/templates/hubble/tls-provided/relay-server-secret.yaml index 2038f785..07059143 100644 --- a/helm/cilium/templates/hubble/tls-provided/relay-server-secret.yaml +++ b/helm/cilium/templates/hubble/tls-provided/relay-server-secret.yaml @@ -4,6 +4,10 @@ kind: Secret metadata: name: hubble-relay-server-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .Values.tls.ca.cert }} diff --git a/helm/cilium/templates/hubble/tls-provided/server-secret.yaml b/helm/cilium/templates/hubble/tls-provided/server-secret.yaml index ecc0bf07..f659a1fb 100644 --- a/helm/cilium/templates/hubble/tls-provided/server-secret.yaml +++ b/helm/cilium/templates/hubble/tls-provided/server-secret.yaml @@ -4,6 +4,10 @@ kind: Secret metadata: name: hubble-server-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .Values.tls.ca.cert }} diff --git a/helm/cilium/templates/hubble/tls-provided/ui-client-certs.yaml b/helm/cilium/templates/hubble/tls-provided/ui-client-certs.yaml index 7ed9472d..ca7683a4 100644 --- a/helm/cilium/templates/hubble/tls-provided/ui-client-certs.yaml +++ b/helm/cilium/templates/hubble/tls-provided/ui-client-certs.yaml @@ -4,6 +4,10 @@ kind: Secret metadata: name: hubble-ui-client-certs namespace: {{ .Release.Namespace }} + {{- with .Values.hubble.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} type: kubernetes.io/tls data: ca.crt: {{ .Values.tls.ca.cert }} diff --git a/helm/cilium/templates/spire/agent/clusterrole.yaml b/helm/cilium/templates/spire/agent/clusterrole.yaml index 57d7ae0d..90e2165d 100644 --- a/helm/cilium/templates/spire/agent/clusterrole.yaml +++ b/helm/cilium/templates/spire/agent/clusterrole.yaml @@ -3,6 +3,10 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} rules: # Required cluster role to allow spire-agent to query k8s API server - apiGroups: [ "" ] diff --git a/helm/cilium/templates/spire/agent/clusterrolebinding.yaml b/helm/cilium/templates/spire/agent/clusterrolebinding.yaml index f7290825..50fe9588 100644 --- a/helm/cilium/templates/spire/agent/clusterrolebinding.yaml +++ b/helm/cilium/templates/spire/agent/clusterrolebinding.yaml @@ -4,6 +4,10 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} roleRef: kind: ClusterRole name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} diff --git a/helm/cilium/templates/spire/agent/configmap.yaml b/helm/cilium/templates/spire/agent/configmap.yaml index 21e1a70a..7d365ba6 100644 --- a/helm/cilium/templates/spire/agent/configmap.yaml +++ b/helm/cilium/templates/spire/agent/configmap.yaml @@ -4,6 +4,10 @@ kind: ConfigMap metadata: name: spire-agent namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: agent.conf: | agent { diff --git a/helm/cilium/templates/spire/agent/daemonset.yaml b/helm/cilium/templates/spire/agent/daemonset.yaml index f186eaef..0b76161e 100644 --- a/helm/cilium/templates/spire/agent/daemonset.yaml +++ b/helm/cilium/templates/spire/agent/daemonset.yaml @@ -4,9 +4,14 @@ kind: DaemonSet metadata: name: spire-agent namespace: {{ .Values.authentication.mutual.spire.install.namespace }} - {{- with .Values.authentication.mutual.spire.install.server.annotations }} + {{- if or .Values.authentication.mutual.spire.install.server.annotations .Values.authentication.mutual.spire.annotations }} annotations: - {{- toYaml . | nindent 8 }} + {{- with .Values.authentication.mutual.spire.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} labels: app: spire-agent @@ -36,7 +41,8 @@ spec: {{- end }} initContainers: - name: init - image: docker.io/library/busybox:1.35.0@sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b + image: {{ include "cilium.image" (list $ .Values.authentication.mutual.spire.install.initImage) | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.initImage.pullPolicy }} command: - /bin/sh - -c @@ -44,7 +50,12 @@ spec: {{- tpl (.Files.Get "files/spire/wait-for-spire.bash") . | nindent 14 }} containers: - name: spire-agent + {{- if eq (typeOf .Values.authentication.mutual.spire.install.agent.image) "string" }} image: {{ .Values.authentication.mutual.spire.install.agent.image }} + {{- else }} + image: {{ include "cilium.image" (list $ .Values.authentication.mutual.spire.install.agent.image) | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.agent.image.pullPolicy }} + {{- end }} args: ["-config", "/run/spire/config/agent.conf"] {{- with .Values.authentication.mutual.spire.install.agent.securityContext }} securityContext: @@ -88,10 +99,12 @@ spec: nodeSelector: {{- toYaml . | nindent 8 }} {{- end }} - {{- with .Values.authentication.mutual.spire.install.agent.tolerations }} tolerations: - {{- toYaml . | trim | nindent 8 }} - {{- end }} + {{- with .Values.authentication.mutual.spire.install.agent.tolerations }} + {{- toYaml . | trim | nindent 8 }} + {{- end }} + - key: {{ .Values.agentNotReadyTaintKey | default "node.cilium.io/agent-not-ready" }} + effect: NoSchedule volumes: - name: spire-config configMap: diff --git a/helm/cilium/templates/spire/agent/serviceaccount.yaml b/helm/cilium/templates/spire/agent/serviceaccount.yaml index f155b5e6..95f7e431 100644 --- a/helm/cilium/templates/spire/agent/serviceaccount.yaml +++ b/helm/cilium/templates/spire/agent/serviceaccount.yaml @@ -4,4 +4,8 @@ kind: ServiceAccount metadata: name: {{ .Values.authentication.mutual.spire.install.agent.serviceAccount.name }} namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/helm/cilium/templates/spire/bundle-configmap.yaml b/helm/cilium/templates/spire/bundle-configmap.yaml index 986e6604..389b8d58 100644 --- a/helm/cilium/templates/spire/bundle-configmap.yaml +++ b/helm/cilium/templates/spire/bundle-configmap.yaml @@ -4,4 +4,8 @@ kind: ConfigMap metadata: name: spire-bundle namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/helm/cilium/templates/spire/namespace.yaml b/helm/cilium/templates/spire/namespace.yaml index a0f3e23d..ccd38680 100644 --- a/helm/cilium/templates/spire/namespace.yaml +++ b/helm/cilium/templates/spire/namespace.yaml @@ -1,6 +1,10 @@ -{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled -}} +{{- if and .Values.authentication.mutual.spire.enabled .Values.authentication.mutual.spire.install.enabled (not .Values.authentication.mutual.spire.install.existingNamespace) -}} apiVersion: v1 kind: Namespace metadata: name: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/helm/cilium/templates/spire/server/clusterrole.yaml b/helm/cilium/templates/spire/server/clusterrole.yaml index 95cc911a..31bc2de1 100644 --- a/helm/cilium/templates/spire/server/clusterrole.yaml +++ b/helm/cilium/templates/spire/server/clusterrole.yaml @@ -4,6 +4,10 @@ kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} rules: # ClusterRole to allow spire-server node attestor to query Token Review API - apiGroups: [ "authentication.k8s.io" ] diff --git a/helm/cilium/templates/spire/server/clusterrolebinding.yaml b/helm/cilium/templates/spire/server/clusterrolebinding.yaml index aaefe4ff..4a04d80c 100644 --- a/helm/cilium/templates/spire/server/clusterrolebinding.yaml +++ b/helm/cilium/templates/spire/server/clusterrolebinding.yaml @@ -3,6 +3,10 @@ kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} roleRef: kind: ClusterRole name: spire-server diff --git a/helm/cilium/templates/spire/server/configmap.yaml b/helm/cilium/templates/spire/server/configmap.yaml index 9785c11e..d7347595 100644 --- a/helm/cilium/templates/spire/server/configmap.yaml +++ b/helm/cilium/templates/spire/server/configmap.yaml @@ -4,6 +4,10 @@ kind: ConfigMap metadata: name: spire-server namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} data: server.conf: | server { diff --git a/helm/cilium/templates/spire/server/role.yaml b/helm/cilium/templates/spire/server/role.yaml index 38e0b113..2d38fe14 100644 --- a/helm/cilium/templates/spire/server/role.yaml +++ b/helm/cilium/templates/spire/server/role.yaml @@ -4,6 +4,10 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} rules: # Role (namespace scoped) to be able to push certificate bundles to a configmap - apiGroups: [ "" ] diff --git a/helm/cilium/templates/spire/server/rolebinding.yaml b/helm/cilium/templates/spire/server/rolebinding.yaml index a3e18731..497bd60c 100644 --- a/helm/cilium/templates/spire/server/rolebinding.yaml +++ b/helm/cilium/templates/spire/server/rolebinding.yaml @@ -4,6 +4,10 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -18,6 +22,10 @@ apiVersion: rbac.authorization.k8s.io/v1 metadata: name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }}-pod namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole diff --git a/helm/cilium/templates/spire/server/service.yaml b/helm/cilium/templates/spire/server/service.yaml index 31fee016..32288a44 100644 --- a/helm/cilium/templates/spire/server/service.yaml +++ b/helm/cilium/templates/spire/server/service.yaml @@ -4,9 +4,14 @@ kind: Service metadata: name: spire-server namespace: {{ .Values.authentication.mutual.spire.install.namespace }} - {{- with .Values.authentication.mutual.spire.install.server.service.annotations }} + {{- if or .Values.authentication.mutual.spire.install.server.service.annotations .Values.authentication.mutual.spire.annotations }} annotations: - {{- toYaml . | nindent 8 }} + {{- with .Values.authentication.mutual.spire.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.service.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} {{- with .Values.authentication.mutual.spire.install.server.service.labels }} labels: diff --git a/helm/cilium/templates/spire/server/serviceaccount.yaml b/helm/cilium/templates/spire/server/serviceaccount.yaml index 2d4aa4a3..cd365c89 100644 --- a/helm/cilium/templates/spire/server/serviceaccount.yaml +++ b/helm/cilium/templates/spire/server/serviceaccount.yaml @@ -4,4 +4,8 @@ kind: ServiceAccount metadata: name: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} namespace: {{ .Values.authentication.mutual.spire.install.namespace }} + {{- with .Values.authentication.mutual.spire.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} diff --git a/helm/cilium/templates/spire/server/statefulset.yaml b/helm/cilium/templates/spire/server/statefulset.yaml index 43c2c9ee..8cc43915 100644 --- a/helm/cilium/templates/spire/server/statefulset.yaml +++ b/helm/cilium/templates/spire/server/statefulset.yaml @@ -4,9 +4,14 @@ kind: StatefulSet metadata: name: spire-server namespace: {{ .Values.authentication.mutual.spire.install.namespace }} - {{- with .Values.authentication.mutual.spire.install.server.annotations }} + {{- if or .Values.authentication.mutual.spire.install.server.annotations .Values.authentication.mutual.spire.annotations }} annotations: - {{- toYaml . | nindent 8 }} + {{- with .Values.authentication.mutual.spire.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.authentication.mutual.spire.install.server.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} {{- end }} labels: app: spire-server @@ -29,20 +34,30 @@ spec: spec: serviceAccountName: {{ .Values.authentication.mutual.spire.install.server.serviceAccount.name }} shareProcessNamespace: true + {{- with .Values.authentication.mutual.spire.install.server.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 8 }} + {{- end }} {{- if gt (len .Values.authentication.mutual.spire.install.server.initContainers) 0 }} initContainers: {{- toYaml .Values.authentication.mutual.spire.install.server.initContainers | nindent 8 }} {{- end }} containers: - name: cilium-init - image: docker.io/library/busybox:1.35.0@sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b + image: {{ include "cilium.image" (list $ .Values.authentication.mutual.spire.install.initImage) | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.initImage.pullPolicy }} command: - /bin/sh - -c - | {{- tpl (.Files.Get "files/spire/init.bash") . | nindent 12 }} - name: spire-server + {{- if eq (typeOf .Values.authentication.mutual.spire.install.server.image) "string" }} image: {{ .Values.authentication.mutual.spire.install.server.image }} + {{- else }} + image: {{ include "cilium.image" (list $ .Values.authentication.mutual.spire.install.server.image) | quote }} + imagePullPolicy: {{ .Values.authentication.mutual.spire.install.server.image.pullPolicy }} + {{- end }} args: - -config - /run/spire/config/server.conf @@ -75,6 +90,10 @@ spec: port: 8080 initialDelaySeconds: 5 periodSeconds: 5 + {{- with .Values.authentication.mutual.spire.install.server.securityContext }} + securityContext: + {{- toYaml . | nindent 10 }} + {{- end }} {{- with .Values.authentication.mutual.spire.install.server.affinity }} affinity: {{- toYaml . | nindent 8 }} diff --git a/helm/cilium/templates/validate.yaml b/helm/cilium/templates/validate.yaml index dfb5cb30..3c89e4e3 100644 --- a/helm/cilium/templates/validate.yaml +++ b/helm/cilium/templates/validate.yaml @@ -30,6 +30,10 @@ {{- end }} {{- end }} +{{- if and .Values.hubble.redact.http.headers.allow .Values.hubble.redact.http.headers.deny }} + {{ fail "Only one of .Values.hubble.redact.http.headers.allow, .Values.hubble.redact.http.headers.deny can be specified"}} +{{- end }} + {{- if and (or .Values.externalWorkloads.enabled .Values.clustermesh.useAPIServer) .Values.clustermesh.apiserver.tls.auto.enabled (eq .Values.clustermesh.apiserver.tls.auto.method "certmanager") }} {{- if not .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef }} {{ fail "ClusterMesh TLS certgen method=certmanager requires that user specifies .Values.clustermesh.apiserver.tls.auto.certManagerIssuerRef" }} @@ -74,3 +78,26 @@ {{ fail "if Cilium Endpoint Slice is enabled (.Values.enableCiliumEndpointSlice=true), it requires .Values.disableEndpointCRD=false" }} {{- end }} {{- end }} + +{{/* validate clustermesh-apiserver */}} +{{- if .Values.clustermesh.useAPIServer }} + {{- if ne .Values.identityAllocationMode "crd" }} + {{ fail (printf "The clustermesh-apiserver cannot be enabled in combination with .Values.identityAllocationMode=%s. To establish a Cluster Mesh, directly configure the parameters to access the remote kvstore through .Values.clustermesh.config" .Values.identityAllocationMode ) }} + {{- end }} + {{- if .Values.disableEndpointCRD }} + {{ fail "The clustermesh-apiserver cannot be enabled in combination with .Values.disableEndpointCRD=true" }} + {{- end }} +{{- end }} +{{- if .Values.externalWorkloads.enabled }} + {{- if ne .Values.identityAllocationMode "crd" }} + {{ fail (printf "External workloads support cannot be enabled in combination with .Values.identityAllocationMode=%s" .Values.identityAllocationMode ) }} + {{- end }} + {{- if .Values.disableEndpointCRD }} + {{ fail "External workloads support cannot be enabled in combination with .Values.disableEndpointCRD=true" }} + {{- end }} +{{- end }} + +{{/*validate ClusterMesh */}} +{{- if and (ne (int .Values.clustermesh.maxConnectedClusters) 255) (ne (int .Values.clustermesh.maxConnectedClusters) 511) }} + {{- fail "max-connected-clusters must be set to 255 or 511" }} +{{- end }} diff --git a/helm/cilium/values.schema.json b/helm/cilium/values.schema.json index 4fbdd99d..57106d3d 100644 --- a/helm/cilium/values.schema.json +++ b/helm/cilium/values.schema.json @@ -64,6 +64,12 @@ "annotateK8sNode": { "type": "boolean" }, + "annotations": { + "type": "object" + }, + "apiRateLimit": { + "type": "string" + }, "authentication": { "type": "object", "properties": { @@ -76,6 +82,9 @@ "mutual": { "type": "object", "properties": { + "connectTimeout": { + "type": "string" + }, "port": { "type": "integer" }, @@ -88,6 +97,9 @@ "agentSocketPath": { "type": "string" }, + "annotations": { + "type": "object" + }, "connectionTimeout": { "type": "string" }, @@ -107,7 +119,27 @@ "type": "object" }, "image": { - "type": "string" + "type": "object", + "properties": { + "digest": { + "type": "string" + }, + "override": { + "type": "string" + }, + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "useDigest": { + "type": "boolean" + } + } }, "labels": { "type": "object" @@ -136,13 +168,50 @@ "type": "boolean" }, "tolerations": { - "type": "array" + "type": "array", + "items": { + "type": "object", + "properties": { + "effect": { + "type": "string" + }, + "key": { + "type": "string" + } + } + } } } }, "enabled": { "type": "boolean" }, + "existingNamespace": { + "type": "boolean" + }, + "initImage": { + "type": "object", + "properties": { + "digest": { + "type": "string" + }, + "override": { + "type": "string" + }, + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "useDigest": { + "type": "boolean" + } + } + }, "namespace": { "type": "string" }, @@ -195,7 +264,27 @@ } }, "image": { - "type": "string" + "type": "object", + "properties": { + "digest": { + "type": "string" + }, + "override": { + "type": "string" + }, + "pullPolicy": { + "type": "string" + }, + "repository": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "useDigest": { + "type": "boolean" + } + } }, "initContainers": { "type": "array" @@ -206,6 +295,12 @@ "nodeSelector": { "type": "object" }, + "podSecurityContext": { + "type": "object" + }, + "securityContext": { + "type": "object" + }, "service": { "type": "object", "properties": { @@ -302,6 +397,17 @@ "properties": { "enabled": { "type": "boolean" + }, + "secretsNamespace": { + "type": "object", + "properties": { + "create": { + "type": "boolean" + }, + "name": { + "type": "string" + } + } } } }, @@ -378,6 +484,9 @@ "certgen": { "type": "object", "properties": { + "affinity": { + "type": "object" + }, "annotations": { "type": "object", "properties": { @@ -465,6 +574,9 @@ "clustermesh": { "type": "object", "properties": { + "annotations": { + "type": "object" + }, "apiserver": { "type": "object", "properties": { @@ -505,31 +617,23 @@ "etcd": { "type": "object", "properties": { - "image": { + "init": { "type": "object", "properties": { - "override": { - "type": "string" - }, - "pullPolicy": { - "type": "string" + "extraArgs": { + "type": "array" }, - "repository": { - "type": "string" + "extraEnv": { + "type": "array" }, - "tag": { - "type": "string" - } - } - }, - "init": { - "type": "object", - "properties": { "resources": { "type": "object" } } }, + "lifecycle": { + "type": "object" + }, "resources": { "type": "object" }, @@ -588,28 +692,8 @@ "extraVolumeMounts": { "type": "array" }, - "image": { - "type": "object", - "properties": { - "digest": { - "type": "string" - }, - "override": { - "type": "string" - }, - "pullPolicy": { - "type": "string" - }, - "repository": { - "type": "string" - }, - "tag": { - "type": "string" - }, - "useDigest": { - "type": "boolean" - } - } + "lifecycle": { + "type": "object" }, "resources": { "type": "object" @@ -635,6 +719,9 @@ } } }, + "lifecycle": { + "type": "object" + }, "metrics": { "type": "object", "properties": { @@ -782,6 +869,9 @@ } } }, + "terminationGracePeriodSeconds": { + "type": "integer" + }, "tls": { "type": "object", "properties": { @@ -816,17 +906,6 @@ } } }, - "ca": { - "type": "object", - "properties": { - "cert": { - "type": "string" - }, - "key": { - "type": "string" - } - } - }, "client": { "type": "object", "properties": { @@ -906,6 +985,9 @@ } } }, + "maxConnectedClusters": { + "type": "integer" + }, "useAPIServer": { "type": "boolean" } @@ -947,6 +1029,22 @@ "logFile": { "type": "string" }, + "resources": { + "type": "object", + "properties": { + "requests": { + "type": "object", + "properties": { + "cpu": { + "type": "string" + }, + "memory": { + "type": "string" + } + } + } + } + }, "uninstall": { "type": "boolean" } @@ -1105,9 +1203,6 @@ "enableCiliumEndpointSlice": { "type": "boolean" }, - "enableCnpStatusUpdates": { - "type": "boolean" - }, "enableCriticalPriorityClass": { "type": "boolean" }, @@ -1123,10 +1218,10 @@ "enableIPv6Masquerade": { "type": "boolean" }, - "enableK8sEventHandover": { + "enableK8sTerminatingEndpoint": { "type": "boolean" }, - "enableK8sTerminatingEndpoint": { + "enableMasqueradeRouteSource": { "type": "boolean" }, "enableRuntimeDeviceDetection": { @@ -1179,12 +1274,29 @@ "secretName": { "type": "string" }, + "strictMode": { + "type": "object", + "properties": { + "allowRemoteNodeIdentities": { + "type": "boolean" + }, + "cidr": { + "type": "string" + }, + "enabled": { + "type": "boolean" + } + } + }, "type": { "type": "string" }, "wireguard": { "type": "object", "properties": { + "persistentKeepalive": { + "type": "string" + }, "userspaceFallback": { "type": "boolean" } @@ -1297,6 +1409,9 @@ } } }, + "annotations": { + "type": "object" + }, "connectTimeoutSeconds": { "type": "integer" }, @@ -1564,6 +1679,9 @@ "etcd": { "type": "object", "properties": { + "annotations": { + "type": "object" + }, "clusterDomain": { "type": "string" }, @@ -1843,9 +1961,85 @@ "hubble": { "type": "object", "properties": { + "annotations": { + "type": "object" + }, "enabled": { "type": "boolean" }, + "export": { + "type": "object", + "properties": { + "dynamic": { + "type": "object", + "properties": { + "config": { + "type": "object", + "properties": { + "configMapName": { + "type": "string" + }, + "content": { + "type": "array", + "items": { + "type": "object", + "properties": { + "excludeFilters": { + "type": "array" + }, + "fieldMask": { + "type": "array" + }, + "filePath": { + "type": "string" + }, + "includeFilters": { + "type": "array" + }, + "name": { + "type": "string" + } + } + } + }, + "createConfigMap": { + "type": "boolean" + } + } + }, + "enabled": { + "type": "boolean" + } + } + }, + "fileMaxBackups": { + "type": "integer" + }, + "fileMaxSizeMb": { + "type": "integer" + }, + "static": { + "type": "object", + "properties": { + "allowList": { + "type": "array" + }, + "denyList": { + "type": "array" + }, + "enabled": { + "type": "boolean" + }, + "fieldMask": { + "type": "array" + }, + "filePath": { + "type": "string" + } + } + } + } + }, "listenAddress": { "type": "string" }, @@ -1896,6 +2090,9 @@ "interval": { "type": "string" }, + "jobLabel": { + "type": "string" + }, "labels": { "type": "object" }, @@ -1940,6 +2137,44 @@ "preferIpv6": { "type": "boolean" }, + "redact": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + }, + "http": { + "type": "object", + "properties": { + "headers": { + "type": "object", + "properties": { + "allow": { + "type": "array" + }, + "deny": { + "type": "array" + } + } + }, + "urlQuery": { + "type": "boolean" + }, + "userInfo": { + "type": "boolean" + } + } + }, + "kafka": { + "type": "object", + "properties": { + "apiKey": { + "type": "boolean" + } + } + } + } + }, "relay": { "type": "object", "properties": { @@ -1977,6 +2212,9 @@ } } }, + "annotations": { + "type": "object" + }, "dialTimeout": { "type": "string" }, @@ -2207,6 +2445,9 @@ }, "mtls": { "type": "boolean" + }, + "relayName": { + "type": "string" } } } @@ -2293,6 +2534,9 @@ "affinity": { "type": "object" }, + "annotations": { + "type": "object" + }, "backend": { "type": "object", "properties": { @@ -2328,6 +2572,22 @@ } } }, + "livenessProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, + "readinessProbe": { + "type": "object", + "properties": { + "enabled": { + "type": "boolean" + } + } + }, "resources": { "type": "object" }, @@ -2591,6 +2851,9 @@ "defaultSecretNamespace": { "type": "string" }, + "enableProxyProtocol": { + "type": "boolean" + }, "enabled": { "type": "boolean" }, @@ -2823,6 +3086,9 @@ "loadBalancer": { "type": "object", "properties": { + "acceleration": { + "type": "string" + }, "l7": { "type": "object", "properties": { @@ -2879,6 +3145,9 @@ "enableHealthCheck": { "type": "boolean" }, + "enableHealthCheckLoadBalancerIP": { + "type": "boolean" + }, "enabled": { "type": "boolean" } @@ -2898,6 +3167,9 @@ "affinity": { "type": "object" }, + "annotations": { + "type": "object" + }, "bootstrapFile": { "type": "string" }, @@ -3073,6 +3345,9 @@ } } }, + "annotations": { + "type": "object" + }, "dashboards": { "type": "object", "properties": { @@ -3227,6 +3502,9 @@ "interval": { "type": "string" }, + "jobLabel": { + "type": "string" + }, "labels": { "type": "object" }, @@ -3330,6 +3608,9 @@ "podSecurityContext": { "type": "object" }, + "policyCIDRMatchMode": { + "type": "array" + }, "policyEnforcementMode": { "type": "string" }, @@ -3384,6 +3665,9 @@ } } }, + "annotations": { + "type": "object" + }, "enabled": { "type": "boolean" }, @@ -3495,6 +3779,12 @@ "prometheus": { "type": "object", "properties": { + "controllerGroupMetrics": { + "type": "array", + "items": { + "type": "string" + } + }, "enabled": { "type": "boolean" }, @@ -3519,6 +3809,9 @@ "interval": { "type": "string" }, + "jobLabel": { + "type": "string" + }, "labels": { "type": "object" }, @@ -3918,6 +4211,9 @@ } } }, + "serviceNoBackendResponse": { + "type": "string" + }, "sleepAfterInit": { "type": "boolean" }, @@ -3999,9 +4295,6 @@ } } }, - "tunnel": { - "type": "string" - }, "tunnelPort": { "type": "integer" }, diff --git a/helm/cilium/values.yaml b/helm/cilium/values.yaml index b5fb5120..b6bb31d6 100644 --- a/helm/cilium/values.yaml +++ b/helm/cilium/values.yaml @@ -1,10 +1,10 @@ # File generated by install/kubernetes/Makefile; DO NOT EDIT. -# This file is based on install/kubernetes/cilium/values.yaml.tmpl. +# This file is based on install/kubernetes/cilium/*values.yaml.tmpl. # upgradeCompatibility helps users upgrading to ensure that the configMap for # Cilium will not change critical values to ensure continued operation -# This is flag is not required for new installations. +# This flag is not required for new installations. # For example: 1.7, 1.8, 1.9 # upgradeCompatibility: '1.8' @@ -15,6 +15,7 @@ debug: # This option is used to enable debug messages for operations related to such # sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is # for enabling debug messages emitted per request, message and connection. + # Multiple values can be set via a space-separated string (e.g. "datapath envoy"). # # Applicable values: # - flow @@ -46,10 +47,12 @@ k8sServicePort: "" # rate limit, the agent and operator will start to throttle requests by delaying # them until there is budget or the request times out. k8sClientRateLimit: - # -- The sustained request rate in requests per second. + # -- (int) The sustained request rate in requests per second. + # @default -- 5 for k8s up to 1.26. 10 for k8s version 1.27+ qps: 5 - # -- The burst request rate in requests per second. + # -- (int) The burst request rate in requests per second. # The rate limiter will allow short bursts with a higher rate. + # @default -- 10 for k8s up to 1.26. 20 for k8s version 1.27+ burst: 10 cluster: @@ -153,7 +156,7 @@ image: registry: gsoci.azurecr.io override: "" repository: "giantswarm/cilium" - tag: "v1.14.5" + tag: "v1.15.1" pullPolicy: "IfNotPresent" # cilium-digest digest: "" @@ -221,6 +224,9 @@ extraConfig: {} # test 2 # test 3 +# -- Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) +annotations: {} + # -- Security Context for cilium-agent pods. podSecurityContext: {} @@ -418,6 +424,12 @@ bgp: bgpControlPlane: # -- Enables the BGP control plane. enabled: false + # -- SecretsNamespace is the namespace which BGP support will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for BGP secrets. + create: false + # -- The name of the secret namespace to which Cilium agents are given read access + name: kube-system pmtuDiscovery: # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to @@ -599,6 +611,12 @@ cni: # inside the agent pod. hostConfDirMountPath: /host/etc/cni/net.d + # -- Specifies the resources for the cni initContainer + resources: + requests: + cpu: 100m + memory: 10Mi + # -- (string) Configure how frequently garbage collection should occur for the datapath # connection tracking table. # @default -- `"0s"` @@ -660,7 +678,8 @@ daemon: # -- Specify which network interfaces can run the eBPF datapath. This means # that a packet sent from a pod to a destination outside the cluster will be # masqueraded (to an output device IPv4 address), if the output device runs the -# program. When not specified, probing will automatically detect devices. +# program. When not specified, probing will automatically detect devices that have +# a non-local route. This should be used only when autodetection is not suitable. # devices: "" # -- Enables experimental support for the detection of new and removed datapath @@ -675,13 +694,6 @@ enableRuntimeDeviceDetection: false # -- Limit iptables-based egress masquerading to interface selector. # egressMasqueradeInterfaces: "" -# -- Whether to enable CNP status updates. -enableCnpStatusUpdates: false - -# -- Configures the use of the KVStore to optimize Kubernetes event handling by -# mirroring it into the KVstore for reduced overhead in large clusters. -enableK8sEventHandover: false - # -- Enable setting identity mark for local traffic. # enableIdentityMark: true @@ -723,8 +735,10 @@ ingressController: # Incoming traffic to http listener will return 308 http error code with respective location in header. enforceHttps: true - # -- IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate - # from Ingress to the Load Balancer service + # -- Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. + enableProxyProtocol: false + + # -- IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] # -- Default secret namespace for ingresses without .spec.tls[].secretName set. @@ -802,6 +816,18 @@ encryption: # This option is only effective when encryption.type is set to "wireguard". nodeEncryption: false + # -- Configure the WireGuard Pod2Pod strict mode. + strictMode: + # -- Enable WireGuard Pod2Pod strict mode. + enabled: false + + # -- CIDR for the WireGuard Pod2Pod strict mode. + cidr: "" + + # -- Allow dynamic lookup of remote node identities. + # This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. + allowRemoteNodeIdentities: false + ipsec: # -- Name of the key file inside the Kubernetes secret configured via secretName. keyFile: "" @@ -826,6 +852,8 @@ encryption: wireguard: # -- Enables the fallback to the user-space implementation. userspaceFallback: false + # -- Controls Wireguard PersistentKeepalive option. Set 0s to disable. + persistentKeepalive: 0s # -- Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. # Name of the key file inside the Kubernetes secret configured via secretName. @@ -967,10 +995,16 @@ certgen: # -- Additional certgen volumeMounts. extraVolumeMounts: [] + # -- Affinity for certgen + affinity: {} + hubble: # -- Enable Hubble (true by default). enabled: true + # -- Annotations to be added to all top-level hubble objects (resources under templates/hubble) + annotations: {} + # -- Buffer size of the channel Hubble uses to receive monitor events. If this # value is not set, the queue size is set to the default monitor queue size. # eventQueueSize: "" @@ -999,7 +1033,7 @@ hubble: # # You can specify the list of metrics from the helm CLI: # - # --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" + # --set hubble.metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" # enabled: "" # -- Enables exporting hubble metrics in OpenMetrics format. @@ -1017,6 +1051,8 @@ hubble: labels: {} # -- Annotations to add to ServiceMonitor hubble annotations: {} + # -- jobLabel to add for ServiceMonitor hubble + jobLabel: "" # -- Interval for scrape metrics. interval: "10s" # -- Relabeling configs for the ServiceMonitor hubble @@ -1040,6 +1076,81 @@ hubble: # -- Unix domain socket path to listen to when Hubble is enabled. socketPath: /var/run/cilium/hubble.sock + # -- Enables redacting sensitive information present in Layer 7 flows. + redact: + enabled: false + http: + # -- Enables redacting URL query (GET) parameters. + # Example: + # + # redact: + # enabled: true + # http: + # urlQuery: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.urlQuery="true" + urlQuery: false + # -- Enables redacting user info, e.g., password when basic auth is used. + # Example: + # + # redact: + # enabled: true + # http: + # userInfo: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.userInfo="true" + userInfo: true + headers: + # -- List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # allow: + # - traceparent + # - tracestate + # - Cache-Control + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" + allow: [] + # -- List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # deny: + # - Authorization + # - Proxy-Authorization + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" + deny: [] + kafka: + # -- Enables redacting Kafka's API key. + # Example: + # + # redact: + # enabled: true + # kafka: + # apiKey: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.kafka.apiKey="true" + apiKey: false + # -- An additional address for Hubble to listen to. # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that # Hubble is listening on port 4244. @@ -1121,7 +1232,7 @@ hubble: image: override: "" repository: "giantswarm/hubble-relay" - tag: "v1.14.5" + tag: "v1.15.1" # hubble-relay-digest digest: "" useDigest: false @@ -1160,6 +1271,9 @@ hubble: # -- Additional hubble-relay environment variables. extraEnv: [] + # -- Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) + annotations: {} + # -- Annotations to be added to hubble-relay pods podAnnotations: {} @@ -1247,6 +1361,12 @@ hubble: extraDnsNames: [] # -- extra IP addresses added to certificate when its auto gen extraIpAddresses: [] + # DNS name used by the backend to connect to the relay + # This is a simple workaround as the relay certificates are currently hardcoded to + # *.hubble-relay.cilium.io + # See https://github.com/cilium/cilium/pull/28709#discussion_r1371792546 + # For GKE Dataplane V2 this should be set to relay.kube-system.svc.cluster.local + relayName: "ui.hubble-relay.cilium.io" # -- Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). dialTimeout: "" @@ -1349,8 +1469,8 @@ hubble: image: override: "" repository: "giantswarm/hubble-ui-backend" - tag: "v0.12.1" - digest: "sha256:1f86f3400827a0451e6332262467f894eeb7caf0eb8779bd951e2caa9d027cbe" + tag: "v0.13.0" + digest: "sha256:1e7657d997c5a48253bb8dc91ecee75b63018d16ff5e5797e5af367336bc8803" useDigest: false pullPolicy: "IfNotPresent" @@ -1366,6 +1486,14 @@ hubble: # -- Additional hubble-ui backend volumeMounts. extraVolumeMounts: [] + livenessProbe: + # -- Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + + readinessProbe: + # -- Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + # -- Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. resources: {} # limits: @@ -1380,8 +1508,8 @@ hubble: image: override: "" repository: "giantswarm/hubble-ui" - tag: "v0.12.1" - digest: "sha256:9e5f81ee747866480ea1ac4630eb6975ff9227f9782b7c93919c081c33f38267" + tag: "v0.13.0" + digest: "sha256:7d663dc16538dd6e29061abd1047013a645e6e69c115e008bee9ea9fef9a6666" useDigest: false pullPolicy: "IfNotPresent" @@ -1413,6 +1541,9 @@ hubble: # -- The number of replicas of Hubble UI to deploy. replicas: 1 + # -- Annotations to be added to all top-level hubble-ui objects (resources under templates/hubble-ui) + annotations: {} + # -- Annotations to be added to hubble-ui pods podAnnotations: {} @@ -1492,6 +1623,55 @@ hubble: # hosts: # - chart-example.local + # -- Hubble flows export. + export: + # --- Defines max file size of output file before it gets rotated. + fileMaxSizeMb: 10 + # --- Defines max number of backup/rotated files. + fileMaxBackups: 5 + # --- Static exporter configuration. + # Static exporter is bound to agent lifecycle. + static: + enabled: false + filePath: /var/run/cilium/hubble/events.log + fieldMask: [] + # - time + # - source + # - destination + # - verdict + allowList: [] + # - '{"verdict":["DROPPED","ERROR"]}' + denyList: [] + # - '{"source_pod":["kube-system/"]}' + # - '{"destination_pod":["kube-system/"]}' + # --- Dynamic exporters configuration. + # Dynamic exporters may be reconfigured without a need of agent restarts. + dynamic: + enabled: false + config: + # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. + configMapName: cilium-flowlog-config + # ---- True if helm installer should create config map. + # Switch to false if you want to self maintain the file content. + createConfigMap: true + # ---- Exporters configuration in YAML format. + content: + - name: all + fieldMask: [] + includeFilters: [] + excludeFilters: [] + filePath: "/var/run/cilium/hubble/events.log" + #- name: "test002" + # filePath: "/var/log/network/flow-log/pa/test002.log" + # fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"] + # includeFilters: + # - source_pod: ["default/"] + # event_type: + # - type: 1 + # - destination_pod: ["frontend/nginx-975996d4c-7hhgt"] + # excludeFilters: [] + # end: "2023-10-09T23:59:59-07:00" + # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" @@ -1542,6 +1722,9 @@ ipam: # @default -- `4.0` externalAPILimitQPS: 4.0 +# -- The api-rate-limit option can be used to overwrite individual settings of the default configuration for rate limiting calls to the Cilium Agent API +apiRateLimit: "" + # -- Configure the eBPF-based ip-masq-agent ipMasqAgent: enabled: false @@ -1640,21 +1823,23 @@ maglev: {} # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. enableIPv4Masquerade: true -# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods -enableIPv6BIGTCP: false +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +enableIPv6Masquerade: true + +# -- Enables masquerading to the source of the route for traffic leaving the node from endpoints. +enableMasqueradeRouteSource: false # -- Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods enableIPv4BIGTCP: false -# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. -enableIPv6Masquerade: true +# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods +enableIPv6BIGTCP: false -# -- Enables egress gateway to redirect and SNAT the traffic that leaves the -# cluster. egressGateway: + # -- Enables egress gateway to redirect and SNAT the traffic that leaves the + # cluster. enabled: false - # -- Install egress gateway IP rules and routes in order to properly steer - # egress gateway traffic to the correct ENI interface + # -- Deprecated without a replacement necessary. installRoutes: false # -- Time between triggers of egress gateway state reconciliations reconciliationTriggerInterval: 1s @@ -1719,8 +1904,11 @@ loadBalancer: # mode: snat # -- acceleration is the option to accelerate service handling via XDP - # e.g. native, disabled - # acceleration: disabled + # Applicable values can be: disabled (do not use XDP), native (XDP BPF + # program is run directly out of the networking driver's early receive + # path), or best-effort (use native mode XDP acceleration on devices + # that support it). + acceleration: disabled # -- dsrDispatch configures whether IP option or IPIP encapsulation is # used to pass a service IP and port to remote backend @@ -1769,6 +1957,10 @@ nodePort: # -- Enable healthcheck nodePort server for NodePort services enableHealthCheck: true + # -- Enable access of the healthcheck nodePort on the LoadBalancerIP. Needs + # EnableHealthCheck to be enabled + enableHealthCheckLoadBalancerIP: false + # policyAuditMode: false # -- The agent can be put into one of the three policy enforcement modes: @@ -1776,6 +1968,10 @@ nodePort: # ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes policyEnforcementMode: "default" +# -- policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. +# The possible value is "nodes". +policyCIDRMatchMode: [] + pprof: # -- Enable pprof for cilium-agent enabled: false @@ -1796,6 +1992,8 @@ prometheus: labels: {} # -- Annotations to add to ServiceMonitor cilium-agent annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-agent + jobLabel: "" # -- Interval for scrape metrics. interval: "10s" # -- Specify the Kubernetes namespace where Prometheus expects to find @@ -1819,6 +2017,15 @@ prometheus: metrics: - +cilium_bpf_map_pressure + # --- Enable controller group metrics for monitoring specific Cilium + # subsystems. The list is a list of controller group names. The special + # values of "all" and "none" are supported. The set of controller + # group names is not guaranteed to be stable between Cilium versions. + controllerGroupMetrics: + - write-cni-file + - sync-host-ips + - sync-lb-maps-with-k8s-services + # -- Grafana dashboards for cilium-agent # grafana can import dashboards based on the label and value # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards @@ -1865,10 +2072,10 @@ envoy: # -- Envoy container image. image: override: "" - repository: "quay.io/cilium/cilium-envoy" - tag: "v1.26.6-ad82c7c56e88989992fd25d8d67747de865c823b" + repository: "giantswarm/cilium-envoy" + tag: "v1.27.3-713b673cccf1af661efd75ca20532336517ddcb9" pullPolicy: "IfNotPresent" - digest: "sha256:992998398dadfff7117bfa9fdb7c9474fefab7f0237263f7c8114e106c67baca" + digest: "sha256:a811830c708296194eaf9cee6c1d22c5f8de3544b7eea6cbdfb810326522a4a2" useDigest: true # -- Additional containers added to the cilium Envoy DaemonSet. @@ -1910,6 +2117,9 @@ envoy: # -- Roll out cilium envoy pods automatically when configmap is updated. rollOutPods: false + # -- Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) + annotations: {} + # -- Security Context for cilium-envoy pods. podSecurityContext: {} @@ -2002,12 +2212,16 @@ envoy: # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy dnsPolicy: "" + # -- Configure Cilium Envoy Prometheus options. + # Note that some of these apply to either cilium-agent or cilium-envoy. prometheus: # -- Enable prometheus metrics for cilium-envoy enabled: true serviceMonitor: # -- Enable service monitors. # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + # Note that this setting applies to both cilium-envoy _and_ cilium-agent + # with Envoy enabled. enabled: false # -- Labels to add to ServiceMonitor cilium-envoy labels: {} @@ -2019,18 +2233,21 @@ envoy: # service monitors configured. # namespace: "" # -- Relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. relabelings: - sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: node replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. metricRelabelings: [] # -- Serve prometheus metrics for cilium-envoy on the configured port port: "9964" # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# Deprecated without replacement in 1.15. To be removed in 1.16. remoteNodeIdentity: true # -- Enable resource quotas for priority classes used in the cluster. @@ -2111,15 +2328,6 @@ tls: # ... # -----END CERTIFICATE----- -# -- Configure the encapsulation configuration for communication between nodes. -# Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. -# Possible values: -# - disabled -# - vxlan -# - geneve -# @default -- `"vxlan"` -tunnel: "" - # -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. # Possible values: # - "" @@ -2140,6 +2348,13 @@ routingMode: "" # @default -- Port 8472 for VXLAN, Port 6081 for Geneve tunnelPort: 0 +# -- Configure what the response should be to traffic for a service without backends. +# "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". +# Possible values: +# - reject (default) +# - drop +serviceNoBackendResponse: reject + # -- Configure the underlying network MTU to overwrite auto-detected MTU. MTU: 0 @@ -2193,6 +2408,9 @@ etcd: nodeSelector: kubernetes.io/os: linux + # -- Annotations to be added to all top-level etcd-operator objects (resources under templates/etcd-operator) + annotations: {} + # -- Security context to be added to cilium-etcd-operator pods podSecurityContext: {} @@ -2261,7 +2479,7 @@ operator: image: override: "" repository: "giantswarm/cilium-operator" - tag: "v1.14.5" + tag: "v1.15.1" # operator-generic-digest genericDigest: "" # operator-azure-digest @@ -2341,6 +2559,9 @@ operator: # -- Additional cilium-operator volumeMounts. extraVolumeMounts: [] + # -- Annotations to be added to all top-level cilium-operator objects (resources under templates/cilium-operator) + annotations: {} + # -- Security context to be added to cilium-operator pods podSecurityContext: {} @@ -2411,6 +2632,8 @@ operator: labels: {} # -- Annotations to add to ServiceMonitor cilium-operator annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-operator + jobLabel: "" # -- Interval for scrape metrics. interval: "10s" # -- Relabeling configs for the ServiceMonitor cilium-operator @@ -2495,6 +2718,9 @@ nodeinit: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + # -- Annotations to be added to all top-level nodeinit objects (resources under templates/cilium-nodeinit) + annotations: {} + # -- Annotations to be added to node-init pods. podAnnotations: {} @@ -2548,7 +2774,7 @@ preflight: image: override: "" repository: "giantswarm/cilium" - tag: "v1.14.5" + tag: "v1.15.1" # cilium-digest digest: "" useDigest: false @@ -2603,6 +2829,9 @@ preflight: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + # -- Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) + annotations: {} + # -- Security context to be added to preflight pods. podSecurityContext: {} @@ -2662,6 +2891,15 @@ enableCriticalPriorityClass: true clustermesh: # -- Deploy clustermesh-apiserver for clustermesh useAPIServer: false + # -- The maximum number of clusters to support in a ClusterMesh. This value + # cannot be changed on running clusters, and all clusters in a ClusterMesh + # must be configured with the same value. Values > 255 will decrease the + # maximum allocatable cluster-local identities. + # Supported values are 255 and 511. + maxConnectedClusters: 255 + + # -- Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) + annotations: {} # -- Clustermesh explicit configuration. config: @@ -2698,19 +2936,16 @@ clustermesh: image: override: "" repository: "giantswarm/cilium-clustermesh-apiserver" - tag: "v1.14.5" + tag: "v1.15.1" # clustermesh-apiserver-digest digest: "" useDigest: false pullPolicy: "IfNotPresent" etcd: - # -- Clustermesh API server etcd image. - image: - override: "" - repository: "giantswarm/etcd" - tag: "v3.5.4@sha256:795d8660c48c439a7c3764c2330ed9222ab5db5bb524d8d0607cac76f7ba82a3" - pullPolicy: "IfNotPresent" + # The etcd binary is included in the clustermesh API server image, so the same image from above is reused. + # Independent override isn't supported, because clustermesh-apiserver is tested against the etcd version it is + # built with. # -- Specifies the resources for etcd container in the apiserver resources: {} @@ -2724,6 +2959,9 @@ clustermesh: # -- Security context to be added to clustermesh-apiserver etcd containers securityContext: {} + # -- lifecycle setting for the etcd container + lifecycle: {} + init: # -- Specifies the resources for etcd init container in the apiserver resources: {} @@ -2734,21 +2972,17 @@ clustermesh: # cpu: 100m # memory: 100Mi + # -- Additional arguments to `clustermesh-apiserver etcdinit`. + extraArgs: [] + + # -- Additional environment variables to `clustermesh-apiserver etcdinit`. + extraEnv: [] + kvstoremesh: # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved # from the remote clusters in the local etcd instance. enabled: false - # -- KVStoreMesh image. - image: - override: "" - repository: "quay.io/cilium/kvstoremesh" - tag: "v1.14.5" - # kvstoremesh-digest - digest: "" - useDigest: false - pullPolicy: "IfNotPresent" - # -- Additional KVStoreMesh arguments. extraArgs: [] @@ -2774,6 +3008,9 @@ clustermesh: drop: - ALL + # -- lifecycle setting for the KVStoreMesh container + lifecycle: {} + service: # -- The type of service used for apiserver access. type: NodePort @@ -2804,6 +3041,12 @@ clustermesh: # -- Number of replicas run for the clustermesh-apiserver deployment. replicas: 1 + # -- lifecycle setting for the apiserver container + lifecycle: {} + + # -- terminationGracePeriodSeconds for the clustermesh-apiserver deployment + terminationGracePeriodSeconds: 30 + # -- Additional clustermesh-apiserver arguments. extraArgs: [] @@ -2946,17 +3189,6 @@ clustermesh: # name: ca-issuer # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. certManagerIssuerRef: {} - # -- Deprecated in favor of tls.ca. To be removed in 1.15. - # base64 encoded PEM values for the ExternalWorkload CA certificate and private key. - ca: - # -- Deprecated in favor of tls.ca.cert. To be removed in 1.15. - # Optional CA cert. If it is provided, it will be used by the 'cronJob' method to - # generate all other certificates. Otherwise, an ephemeral CA is generated. - cert: "" - # -- Deprecated in favor of tls.ca.key. To be removed in 1.15. - # Optional CA private key. If it is provided, it will be used by the 'cronJob' method to - # generate all other certificates. Otherwise, an ephemeral CA is generated. - key: "" # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. # Used if 'auto' is not enabled. server: @@ -2997,7 +3229,7 @@ clustermesh: etcd: # -- Enables exporting etcd metrics in OpenMetrics format. - enabled: false + enabled: true # -- Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. mode: basic # -- Configure the port the etcd metric server listens on. @@ -3097,6 +3329,8 @@ dnsProxy: proxyPort: 0 # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. proxyResponseMaxDelay: 100ms + # -- DNS proxy operation mode (true/false, or unset to use version dependent defaults) + # enableTransparentMode: true # -- SCTP Configuration Values sctp: @@ -3121,10 +3355,14 @@ authentication: mutual: # -- Port on the agent where mutual authentication handshakes between agents will be performed port: 4250 + # -- Timeout for connecting to the remote node TCP socket + connectTimeout: 5s # Settings for SPIRE spire: # -- Enable SPIRE integration (beta) enabled: false + # -- Annotations to be added to all top-level spire objects (resources under templates/spire) + annotations: {} # Settings to control the SPIRE installation and configuration install: # -- Enable SPIRE installation. @@ -3132,10 +3370,26 @@ authentication: enabled: true # -- SPIRE namespace to install into namespace: cilium-spire + # -- SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. + existingNamespace: false + # -- init container image of SPIRE agent and server + initImage: + override: "" + repository: "docker.io/library/busybox" + tag: "1.36.1" + digest: "sha256:223ae047b1065bd069aac01ae3ac8088b3ca4a527827e283b85112f29385fb1b" + useDigest: true + pullPolicy: "IfNotPresent" # SPIRE agent configuration agent: # -- SPIRE agent image - image: ghcr.io/spiffe/spire-agent:1.6.3@sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823 + image: + override: "" + repository: "ghcr.io/spiffe/spire-agent" + tag: "1.8.5" + digest: "sha256:99405637647968245ff9fe215f8bd2bd0ea9807be9725f8bf19fe1b21471e52b" + useDigest: true + pullPolicy: "IfNotPresent" # -- SPIRE agent service account serviceAccount: create: true @@ -3147,8 +3401,21 @@ authentication: # -- SPIRE Workload Attestor kubelet verification. skipKubeletVerification: true # -- SPIRE agent tolerations configuration + # By default it follows the same tolerations as the agent itself + # to allow the Cilium agent on this node to connect to SPIRE. # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - tolerations: [] + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" # -- SPIRE agent affinity configuration affinity: {} # -- SPIRE agent nodeSelector configuration @@ -3164,7 +3431,13 @@ authentication: securityContext: {} server: # -- SPIRE server image - image: ghcr.io/spiffe/spire-server:1.6.3@sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f + image: + override: "" + repository: "ghcr.io/spiffe/spire-server" + tag: "1.8.5" + digest: "sha256:28269265882048dcf0fed32fe47663cd98613727210b8d1a55618826f9bf5428" + useDigest: true + pullPolicy: "IfNotPresent" # -- SPIRE server service account serviceAccount: create: true @@ -3201,6 +3474,14 @@ authentication: accessMode: ReadWriteOnce # -- StorageClass of the SPIRE server data storage storageClass: "" + # -- Security context to be added to spire server pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire server containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} # SPIRE CA configuration ca: # -- SPIRE CA key type diff --git a/helm/cilium/values.yaml.tmpl b/helm/cilium/values.yaml.tmpl index 34b33be7..75a88198 100644 --- a/helm/cilium/values.yaml.tmpl +++ b/helm/cilium/values.yaml.tmpl @@ -1,7 +1,7 @@ # upgradeCompatibility helps users upgrading to ensure that the configMap for # Cilium will not change critical values to ensure continued operation -# This is flag is not required for new installations. +# This flag is not required for new installations. # For example: 1.7, 1.8, 1.9 # upgradeCompatibility: '1.8' @@ -12,6 +12,7 @@ debug: # This option is used to enable debug messages for operations related to such # sub-system such as (e.g. kvstore, envoy, datapath or policy), and flow is # for enabling debug messages emitted per request, message and connection. + # Multiple values can be set via a space-separated string (e.g. "datapath envoy"). # # Applicable values: # - flow @@ -43,10 +44,12 @@ k8sServicePort: "" # rate limit, the agent and operator will start to throttle requests by delaying # them until there is budget or the request times out. k8sClientRateLimit: - # -- The sustained request rate in requests per second. + # -- (int) The sustained request rate in requests per second. + # @default -- 5 for k8s up to 1.26. 10 for k8s version 1.27+ qps: 5 - # -- The burst request rate in requests per second. + # -- (int) The burst request rate in requests per second. # The rate limiter will allow short bursts with a higher rate. + # @default -- 10 for k8s up to 1.26. 20 for k8s version 1.27+ burst: 10 cluster: @@ -218,6 +221,9 @@ extraConfig: {} # test 2 # test 3 +# -- Annotations to be added to all top-level cilium-agent objects (resources under templates/cilium-agent) +annotations: {} + # -- Security Context for cilium-agent pods. podSecurityContext: {} @@ -415,6 +421,12 @@ bgp: bgpControlPlane: # -- Enables the BGP control plane. enabled: false + # -- SecretsNamespace is the namespace which BGP support will retrieve secrets from. + secretsNamespace: + # -- Create secrets namespace for BGP secrets. + create: false + # -- The name of the secret namespace to which Cilium agents are given read access + name: kube-system pmtuDiscovery: # -- Enable path MTU discovery to send ICMP fragmentation-needed replies to @@ -596,6 +608,12 @@ cni: # inside the agent pod. hostConfDirMountPath: /host/etc/cni/net.d + # -- Specifies the resources for the cni initContainer + resources: + requests: + cpu: 100m + memory: 10Mi + # -- (string) Configure how frequently garbage collection should occur for the datapath # connection tracking table. # @default -- `"0s"` @@ -657,7 +675,8 @@ daemon: # -- Specify which network interfaces can run the eBPF datapath. This means # that a packet sent from a pod to a destination outside the cluster will be # masqueraded (to an output device IPv4 address), if the output device runs the -# program. When not specified, probing will automatically detect devices. +# program. When not specified, probing will automatically detect devices that have +# a non-local route. This should be used only when autodetection is not suitable. # devices: "" # -- Enables experimental support for the detection of new and removed datapath @@ -672,13 +691,6 @@ enableRuntimeDeviceDetection: false # -- Limit iptables-based egress masquerading to interface selector. # egressMasqueradeInterfaces: "" -# -- Whether to enable CNP status updates. -enableCnpStatusUpdates: false - -# -- Configures the use of the KVStore to optimize Kubernetes event handling by -# mirroring it into the KVstore for reduced overhead in large clusters. -enableK8sEventHandover: false - # -- Enable setting identity mark for local traffic. # enableIdentityMark: true @@ -720,8 +732,10 @@ ingressController: # Incoming traffic to http listener will return 308 http error code with respective location in header. enforceHttps: true - # -- IngressLBAnnotations are the annotation prefixes, which are used to filter annotations to propagate - # from Ingress to the Load Balancer service + # -- Enable proxy protocol for all Ingress listeners. Note that _only_ Proxy protocol traffic will be accepted once this is enabled. + enableProxyProtocol: false + + # -- IngressLBAnnotations are the annotation and label prefixes, which are used to filter annotations and/or labels to propagate from Ingress to the Load Balancer service ingressLBAnnotationPrefixes: ['service.beta.kubernetes.io', 'service.kubernetes.io', 'cloud.google.com'] # -- Default secret namespace for ingresses without .spec.tls[].secretName set. @@ -799,6 +813,18 @@ encryption: # This option is only effective when encryption.type is set to "wireguard". nodeEncryption: false + # -- Configure the WireGuard Pod2Pod strict mode. + strictMode: + # -- Enable WireGuard Pod2Pod strict mode. + enabled: false + + # -- CIDR for the WireGuard Pod2Pod strict mode. + cidr: "" + + # -- Allow dynamic lookup of remote node identities. + # This is required when tunneling is used or direct routing is used and the node CIDR and pod CIDR overlap. + allowRemoteNodeIdentities: false + ipsec: # -- Name of the key file inside the Kubernetes secret configured via secretName. keyFile: "" @@ -823,6 +849,8 @@ encryption: wireguard: # -- Enables the fallback to the user-space implementation. userspaceFallback: false + # -- Controls Wireguard PersistentKeepalive option. Set 0s to disable. + persistentKeepalive: 0s # -- Deprecated in favor of encryption.ipsec.keyFile. To be removed in 1.15. # Name of the key file inside the Kubernetes secret configured via secretName. @@ -964,10 +992,16 @@ certgen: # -- Additional certgen volumeMounts. extraVolumeMounts: [] + # -- Affinity for certgen + affinity: {} + hubble: # -- Enable Hubble (true by default). enabled: true + # -- Annotations to be added to all top-level hubble objects (resources under templates/hubble) + annotations: {} + # -- Buffer size of the channel Hubble uses to receive monitor events. If this # value is not set, the queue size is set to the default monitor queue size. # eventQueueSize: "" @@ -996,7 +1030,7 @@ hubble: # # You can specify the list of metrics from the helm CLI: # - # --set metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" + # --set hubble.metrics.enabled="{dns:query;ignoreAAAA,drop,tcp,flow,icmp,http}" # enabled: "" # -- Enables exporting hubble metrics in OpenMetrics format. @@ -1014,6 +1048,8 @@ hubble: labels: {} # -- Annotations to add to ServiceMonitor hubble annotations: {} + # -- jobLabel to add for ServiceMonitor hubble + jobLabel: "" # -- Interval for scrape metrics. interval: "10s" # -- Relabeling configs for the ServiceMonitor hubble @@ -1037,6 +1073,81 @@ hubble: # -- Unix domain socket path to listen to when Hubble is enabled. socketPath: /var/run/cilium/hubble.sock + # -- Enables redacting sensitive information present in Layer 7 flows. + redact: + enabled: false + http: + # -- Enables redacting URL query (GET) parameters. + # Example: + # + # redact: + # enabled: true + # http: + # urlQuery: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.urlQuery="true" + urlQuery: false + # -- Enables redacting user info, e.g., password when basic auth is used. + # Example: + # + # redact: + # enabled: true + # http: + # userInfo: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.userInfo="true" + userInfo: true + headers: + # -- List of HTTP headers to allow: headers not matching will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # allow: + # - traceparent + # - tracestate + # - Cache-Control + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.allow="traceparent,tracestate,Cache-Control" + allow: [] + # -- List of HTTP headers to deny: matching headers will be redacted. Note: `allow` and `deny` lists cannot be used both at the same time, only one can be present. + # Example: + # redact: + # enabled: true + # http: + # headers: + # deny: + # - Authorization + # - Proxy-Authorization + # + # You can specify the options from the helm CLI: + # --set hubble.redact.enabled="true" + # --set hubble.redact.http.headers.deny="Authorization,Proxy-Authorization" + deny: [] + kafka: + # -- Enables redacting Kafka's API key. + # Example: + # + # redact: + # enabled: true + # kafka: + # apiKey: true + # + # You can specify the options from the helm CLI: + # + # --set hubble.redact.enabled="true" + # --set hubble.redact.kafka.apiKey="true" + apiKey: false + # -- An additional address for Hubble to listen to. # Set this field ":4244" if you are enabling Hubble Relay, as it assumes that # Hubble is listening on port 4244. @@ -1157,6 +1268,9 @@ hubble: # -- Additional hubble-relay environment variables. extraEnv: [] + # -- Annotations to be added to all top-level hubble-relay objects (resources under templates/hubble-relay) + annotations: {} + # -- Annotations to be added to hubble-relay pods podAnnotations: {} @@ -1244,6 +1358,12 @@ hubble: extraDnsNames: [] # -- extra IP addresses added to certificate when its auto gen extraIpAddresses: [] + # DNS name used by the backend to connect to the relay + # This is a simple workaround as the relay certificates are currently hardcoded to + # *.hubble-relay.cilium.io + # See https://github.com/cilium/cilium/pull/28709#discussion_r1371792546 + # For GKE Dataplane V2 this should be set to relay.kube-system.svc.cluster.local + relayName: "ui.hubble-relay.cilium.io" # -- Dial timeout to connect to the local hubble instance to receive peer information (e.g. "30s"). dialTimeout: "" @@ -1363,6 +1483,14 @@ hubble: # -- Additional hubble-ui backend volumeMounts. extraVolumeMounts: [] + livenessProbe: + # -- Enable liveness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + + readinessProbe: + # -- Enable readiness probe for Hubble-ui backend (requires Hubble-ui 0.12+) + enabled: false + # -- Resource requests and limits for the 'backend' container of the 'hubble-ui' deployment. resources: {} # limits: @@ -1410,6 +1538,9 @@ hubble: # -- The number of replicas of Hubble UI to deploy. replicas: 1 + # -- Annotations to be added to all top-level hubble-ui objects (resources under templates/hubble-ui) + annotations: {} + # -- Annotations to be added to hubble-ui pods podAnnotations: {} @@ -1489,6 +1620,55 @@ hubble: # hosts: # - chart-example.local + # -- Hubble flows export. + export: + # --- Defines max file size of output file before it gets rotated. + fileMaxSizeMb: 10 + # --- Defines max number of backup/rotated files. + fileMaxBackups: 5 + # --- Static exporter configuration. + # Static exporter is bound to agent lifecycle. + static: + enabled: false + filePath: /var/run/cilium/hubble/events.log + fieldMask: [] + # - time + # - source + # - destination + # - verdict + allowList: [] + # - '{"verdict":["DROPPED","ERROR"]}' + denyList: [] + # - '{"source_pod":["kube-system/"]}' + # - '{"destination_pod":["kube-system/"]}' + # --- Dynamic exporters configuration. + # Dynamic exporters may be reconfigured without a need of agent restarts. + dynamic: + enabled: false + config: + # ---- Name of configmap with configuration that may be altered to reconfigure exporters within a running agents. + configMapName: cilium-flowlog-config + # ---- True if helm installer should create config map. + # Switch to false if you want to self maintain the file content. + createConfigMap: true + # ---- Exporters configuration in YAML format. + content: + - name: all + fieldMask: [] + includeFilters: [] + excludeFilters: [] + filePath: "/var/run/cilium/hubble/events.log" + #- name: "test002" + # filePath: "/var/log/network/flow-log/pa/test002.log" + # fieldMask: ["source.namespace", "source.pod_name", "destination.namespace", "destination.pod_name", "verdict"] + # includeFilters: + # - source_pod: ["default/"] + # event_type: + # - type: 1 + # - destination_pod: ["frontend/nginx-975996d4c-7hhgt"] + # excludeFilters: [] + # end: "2023-10-09T23:59:59-07:00" + # -- Method to use for identity allocation (`crd` or `kvstore`). identityAllocationMode: "crd" @@ -1539,6 +1719,9 @@ ipam: # @default -- `4.0` externalAPILimitQPS: 4.0 +# -- The api-rate-limit option can be used to overwrite individual settings of the default configuration for rate limiting calls to the Cilium Agent API +apiRateLimit: "" + # -- Configure the eBPF-based ip-masq-agent ipMasqAgent: enabled: false @@ -1637,21 +1820,23 @@ maglev: {} # -- Enables masquerading of IPv4 traffic leaving the node from endpoints. enableIPv4Masquerade: true -# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods -enableIPv6BIGTCP: false +# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. +enableIPv6Masquerade: true + +# -- Enables masquerading to the source of the route for traffic leaving the node from endpoints. +enableMasqueradeRouteSource: false # -- Enables IPv4 BIG TCP support which increases maximum IPv4 GSO/GRO limits for nodes and pods enableIPv4BIGTCP: false -# -- Enables masquerading of IPv6 traffic leaving the node from endpoints. -enableIPv6Masquerade: true +# -- Enables IPv6 BIG TCP support which increases maximum IPv6 GSO/GRO limits for nodes and pods +enableIPv6BIGTCP: false -# -- Enables egress gateway to redirect and SNAT the traffic that leaves the -# cluster. egressGateway: + # -- Enables egress gateway to redirect and SNAT the traffic that leaves the + # cluster. enabled: false - # -- Install egress gateway IP rules and routes in order to properly steer - # egress gateway traffic to the correct ENI interface + # -- Deprecated without a replacement necessary. installRoutes: false # -- Time between triggers of egress gateway state reconciliations reconciliationTriggerInterval: 1s @@ -1716,8 +1901,11 @@ loadBalancer: # mode: snat # -- acceleration is the option to accelerate service handling via XDP - # e.g. native, disabled - # acceleration: disabled + # Applicable values can be: disabled (do not use XDP), native (XDP BPF + # program is run directly out of the networking driver's early receive + # path), or best-effort (use native mode XDP acceleration on devices + # that support it). + acceleration: disabled # -- dsrDispatch configures whether IP option or IPIP encapsulation is # used to pass a service IP and port to remote backend @@ -1766,6 +1954,10 @@ nodePort: # -- Enable healthcheck nodePort server for NodePort services enableHealthCheck: true + # -- Enable access of the healthcheck nodePort on the LoadBalancerIP. Needs + # EnableHealthCheck to be enabled + enableHealthCheckLoadBalancerIP: false + # policyAuditMode: false # -- The agent can be put into one of the three policy enforcement modes: @@ -1773,6 +1965,10 @@ nodePort: # ref: https://docs.cilium.io/en/stable/security/policy/intro/#policy-enforcement-modes policyEnforcementMode: "default" +# -- policyCIDRMatchMode is a list of entities that may be selected by CIDR selector. +# The possible value is "nodes". +policyCIDRMatchMode: [] + pprof: # -- Enable pprof for cilium-agent enabled: false @@ -1793,6 +1989,8 @@ prometheus: labels: {} # -- Annotations to add to ServiceMonitor cilium-agent annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-agent + jobLabel: "" # -- Interval for scrape metrics. interval: "10s" # -- Specify the Kubernetes namespace where Prometheus expects to find @@ -1816,6 +2014,15 @@ prometheus: metrics: - +cilium_bpf_map_pressure + # --- Enable controller group metrics for monitoring specific Cilium + # subsystems. The list is a list of controller group names. The special + # values of "all" and "none" are supported. The set of controller + # group names is not guaranteed to be stable between Cilium versions. + controllerGroupMetrics: + - write-cni-file + - sync-host-ips + - sync-lb-maps-with-k8s-services + # -- Grafana dashboards for cilium-agent # grafana can import dashboards based on the label and value # ref: https://github.com/grafana/helm-charts/tree/main/charts/grafana#sidecar-for-dashboards @@ -1862,10 +2069,10 @@ envoy: # -- Envoy container image. image: override: "" - repository: "quay.io/cilium/cilium-envoy" - tag: "v1.26.6-ad82c7c56e88989992fd25d8d67747de865c823b" + repository: "${CILIUM_ENVOY_REPO}" + tag: "${CILIUM_ENVOY_VERSION}" pullPolicy: "${PULL_POLICY}" - digest: "sha256:992998398dadfff7117bfa9fdb7c9474fefab7f0237263f7c8114e106c67baca" + digest: "${CILIUM_ENVOY_DIGEST}" useDigest: true # -- Additional containers added to the cilium Envoy DaemonSet. @@ -1907,6 +2114,9 @@ envoy: # -- Roll out cilium envoy pods automatically when configmap is updated. rollOutPods: false + # -- Annotations to be added to all top-level cilium-envoy objects (resources under templates/cilium-envoy) + annotations: {} + # -- Security Context for cilium-envoy pods. podSecurityContext: {} @@ -1999,12 +2209,16 @@ envoy: # Ref: https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/#pod-s-dns-policy dnsPolicy: "" + # -- Configure Cilium Envoy Prometheus options. + # Note that some of these apply to either cilium-agent or cilium-envoy. prometheus: # -- Enable prometheus metrics for cilium-envoy enabled: true serviceMonitor: # -- Enable service monitors. # This requires the prometheus CRDs to be available (see https://github.com/prometheus-operator/prometheus-operator/blob/main/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml) + # Note that this setting applies to both cilium-envoy _and_ cilium-agent + # with Envoy enabled. enabled: false # -- Labels to add to ServiceMonitor cilium-envoy labels: {} @@ -2016,18 +2230,21 @@ envoy: # service monitors configured. # namespace: "" # -- Relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. relabelings: - sourceLabels: - __meta_kubernetes_pod_node_name targetLabel: node replacement: ${1} # -- Metrics relabeling configs for the ServiceMonitor cilium-envoy + # or for cilium-agent with Envoy configured. metricRelabelings: [] # -- Serve prometheus metrics for cilium-envoy on the configured port port: "9964" # -- Enable use of the remote node identity. # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity +# Deprecated without replacement in 1.15. To be removed in 1.16. remoteNodeIdentity: true # -- Enable resource quotas for priority classes used in the cluster. @@ -2108,15 +2325,6 @@ tls: # ... # -----END CERTIFICATE----- -# -- Configure the encapsulation configuration for communication between nodes. -# Deprecated in favor of tunnelProtocol and routingMode. To be removed in 1.15. -# Possible values: -# - disabled -# - vxlan -# - geneve -# @default -- `"vxlan"` -tunnel: "" - # -- Tunneling protocol to use in tunneling mode and for ad-hoc tunnels. # Possible values: # - "" @@ -2137,6 +2345,13 @@ routingMode: "" # @default -- Port 8472 for VXLAN, Port 6081 for Geneve tunnelPort: 0 +# -- Configure what the response should be to traffic for a service without backends. +# "reject" only works on kernels >= 5.10, on lower kernels we fallback to "drop". +# Possible values: +# - reject (default) +# - drop +serviceNoBackendResponse: reject + # -- Configure the underlying network MTU to overwrite auto-detected MTU. MTU: 0 @@ -2190,6 +2405,9 @@ etcd: nodeSelector: kubernetes.io/os: linux + # -- Annotations to be added to all top-level etcd-operator objects (resources under templates/etcd-operator) + annotations: {} + # -- Security context to be added to cilium-etcd-operator pods podSecurityContext: {} @@ -2338,6 +2556,9 @@ operator: # -- Additional cilium-operator volumeMounts. extraVolumeMounts: [] + # -- Annotations to be added to all top-level cilium-operator objects (resources under templates/cilium-operator) + annotations: {} + # -- Security context to be added to cilium-operator pods podSecurityContext: {} @@ -2408,6 +2629,8 @@ operator: labels: {} # -- Annotations to add to ServiceMonitor cilium-operator annotations: {} + # -- jobLabel to add for ServiceMonitor cilium-operator + jobLabel: "" # -- Interval for scrape metrics. interval: "10s" # -- Relabeling configs for the ServiceMonitor cilium-operator @@ -2492,6 +2715,9 @@ nodeinit: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + # -- Annotations to be added to all top-level nodeinit objects (resources under templates/cilium-nodeinit) + annotations: {} + # -- Annotations to be added to node-init pods. podAnnotations: {} @@ -2600,6 +2826,9 @@ preflight: # value: "value" # effect: "NoSchedule|PreferNoSchedule|NoExecute(1.6 only)" + # -- Annotations to be added to all top-level preflight objects (resources under templates/cilium-preflight) + annotations: {} + # -- Security context to be added to preflight pods. podSecurityContext: {} @@ -2659,6 +2888,15 @@ enableCriticalPriorityClass: true clustermesh: # -- Deploy clustermesh-apiserver for clustermesh useAPIServer: false + # -- The maximum number of clusters to support in a ClusterMesh. This value + # cannot be changed on running clusters, and all clusters in a ClusterMesh + # must be configured with the same value. Values > 255 will decrease the + # maximum allocatable cluster-local identities. + # Supported values are 255 and 511. + maxConnectedClusters: 255 + + # -- Annotations to be added to all top-level clustermesh objects (resources under templates/clustermesh-apiserver and templates/clustermesh-config) + annotations: {} # -- Clustermesh explicit configuration. config: @@ -2702,12 +2940,9 @@ clustermesh: pullPolicy: "${PULL_POLICY}" etcd: - # -- Clustermesh API server etcd image. - image: - override: "" - repository: "${ETCD_REPO}" - tag: "${ETCD_VERSION}" - pullPolicy: "${PULL_POLICY}" + # The etcd binary is included in the clustermesh API server image, so the same image from above is reused. + # Independent override isn't supported, because clustermesh-apiserver is tested against the etcd version it is + # built with. # -- Specifies the resources for etcd container in the apiserver resources: {} @@ -2721,6 +2956,9 @@ clustermesh: # -- Security context to be added to clustermesh-apiserver etcd containers securityContext: {} + # -- lifecycle setting for the etcd container + lifecycle: {} + init: # -- Specifies the resources for etcd init container in the apiserver resources: {} @@ -2731,21 +2969,17 @@ clustermesh: # cpu: 100m # memory: 100Mi + # -- Additional arguments to `clustermesh-apiserver etcdinit`. + extraArgs: [] + + # -- Additional environment variables to `clustermesh-apiserver etcdinit`. + extraEnv: [] + kvstoremesh: # -- Enable KVStoreMesh. KVStoreMesh caches the information retrieved # from the remote clusters in the local etcd instance. enabled: false - # -- KVStoreMesh image. - image: - override: "" - repository: "${KVSTOREMESH_REPO}" - tag: "${CILIUM_VERSION}" - # kvstoremesh-digest - digest: ${KVSTOREMESH_DIGEST} - useDigest: ${USE_DIGESTS} - pullPolicy: "${PULL_POLICY}" - # -- Additional KVStoreMesh arguments. extraArgs: [] @@ -2771,6 +3005,9 @@ clustermesh: drop: - ALL + # -- lifecycle setting for the KVStoreMesh container + lifecycle: {} + service: # -- The type of service used for apiserver access. type: NodePort @@ -2801,6 +3038,12 @@ clustermesh: # -- Number of replicas run for the clustermesh-apiserver deployment. replicas: 1 + # -- lifecycle setting for the apiserver container + lifecycle: {} + + # -- terminationGracePeriodSeconds for the clustermesh-apiserver deployment + terminationGracePeriodSeconds: 30 + # -- Additional clustermesh-apiserver arguments. extraArgs: [] @@ -2943,17 +3186,6 @@ clustermesh: # name: ca-issuer # -- certmanager issuer used when clustermesh.apiserver.tls.auto.method=certmanager. certManagerIssuerRef: {} - # -- Deprecated in favor of tls.ca. To be removed in 1.15. - # base64 encoded PEM values for the ExternalWorkload CA certificate and private key. - ca: - # -- Deprecated in favor of tls.ca.cert. To be removed in 1.15. - # Optional CA cert. If it is provided, it will be used by the 'cronJob' method to - # generate all other certificates. Otherwise, an ephemeral CA is generated. - cert: "" - # -- Deprecated in favor of tls.ca.key. To be removed in 1.15. - # Optional CA private key. If it is provided, it will be used by the 'cronJob' method to - # generate all other certificates. Otherwise, an ephemeral CA is generated. - key: "" # -- base64 encoded PEM values for the clustermesh-apiserver server certificate and private key. # Used if 'auto' is not enabled. server: @@ -2994,7 +3226,7 @@ clustermesh: etcd: # -- Enables exporting etcd metrics in OpenMetrics format. - enabled: false + enabled: true # -- Set level of detail for etcd metrics; specify 'extensive' to include server side gRPC histogram metrics. mode: basic # -- Configure the port the etcd metric server listens on. @@ -3094,6 +3326,8 @@ dnsProxy: proxyPort: 0 # -- The maximum time the DNS proxy holds an allowed DNS response before sending it along. Responses are sent as soon as the datapath is updated with the new IP information. proxyResponseMaxDelay: 100ms + # -- DNS proxy operation mode (true/false, or unset to use version dependent defaults) + # enableTransparentMode: true # -- SCTP Configuration Values sctp: @@ -3118,10 +3352,14 @@ authentication: mutual: # -- Port on the agent where mutual authentication handshakes between agents will be performed port: 4250 + # -- Timeout for connecting to the remote node TCP socket + connectTimeout: 5s # Settings for SPIRE spire: # -- Enable SPIRE integration (beta) enabled: false + # -- Annotations to be added to all top-level spire objects (resources under templates/spire) + annotations: {} # Settings to control the SPIRE installation and configuration install: # -- Enable SPIRE installation. @@ -3129,10 +3367,26 @@ authentication: enabled: true # -- SPIRE namespace to install into namespace: cilium-spire + # -- SPIRE namespace already exists. Set to true if Helm should not create, manage, and import the SPIRE namespace. + existingNamespace: false + # -- init container image of SPIRE agent and server + initImage: + override: "" + repository: "${SPIRE_INIT_REPO}" + tag: "${SPIRE_INIT_VERSION}" + digest: "${SPIRE_INIT_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" # SPIRE agent configuration agent: # -- SPIRE agent image - image: ghcr.io/spiffe/spire-agent:1.6.3@sha256:8eef9857bf223181ecef10d9bbcd2f7838f3689e9bd2445bede35066a732e823 + image: + override: "" + repository: "${SPIRE_AGENT_REPO}" + tag: "${SPIRE_AGENT_VERSION}" + digest: "${SPIRE_AGENT_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" # -- SPIRE agent service account serviceAccount: create: true @@ -3144,8 +3398,21 @@ authentication: # -- SPIRE Workload Attestor kubelet verification. skipKubeletVerification: true # -- SPIRE agent tolerations configuration + # By default it follows the same tolerations as the agent itself + # to allow the Cilium agent on this node to connect to SPIRE. # ref: https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/ - tolerations: [] + tolerations: + - key: node.kubernetes.io/not-ready + effect: NoSchedule + - key: node-role.kubernetes.io/master + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + effect: NoSchedule + - key: node.cloudprovider.kubernetes.io/uninitialized + effect: NoSchedule + value: "true" + - key: CriticalAddonsOnly + operator: "Exists" # -- SPIRE agent affinity configuration affinity: {} # -- SPIRE agent nodeSelector configuration @@ -3161,7 +3428,13 @@ authentication: securityContext: {} server: # -- SPIRE server image - image: ghcr.io/spiffe/spire-server:1.6.3@sha256:f4bc49fb0bd1d817a6c46204cc7ce943c73fb0a5496a78e0e4dc20c9a816ad7f + image: + override: "" + repository: "${SPIRE_SERVER_REPO}" + tag: "${SPIRE_SERVER_VERSION}" + digest: "${SPIRE_SERVER_DIGEST}" + useDigest: true + pullPolicy: "${PULL_POLICY}" # -- SPIRE server service account serviceAccount: create: true @@ -3198,6 +3471,14 @@ authentication: accessMode: ReadWriteOnce # -- StorageClass of the SPIRE server data storage storageClass: "" + # -- Security context to be added to spire server pods. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-pod + podSecurityContext: {} + # -- Security context to be added to spire server containers. + # SecurityContext holds pod-level security attributes and common container settings. + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/#set-the-security-context-for-a-container + securityContext: {} # SPIRE CA configuration ca: # -- SPIRE CA key type diff --git a/vendir.lock.yml b/vendir.lock.yml index 125115b2..4cbeffbf 100644 --- a/vendir.lock.yml +++ b/vendir.lock.yml @@ -2,10 +2,10 @@ apiVersion: vendir.k14s.io/v1alpha1 directories: - contents: - git: - commitTitle: 'Merge pull request #5 from giantswarm/gs-remove-nulls-helm-values...' - sha: 31780812ec5e3b10d69899f0cc1020220d05754e + commitTitle: Fix hubble-ui retagged image (#13) + sha: 9995ffa820c0e829b80e89be5904455df9f2b447 tags: - - v1.14.3-264-g31780812ec + - 1.15.1-37-g9995ffa820 path: cilium path: vendor - contents: diff --git a/vendir.yml b/vendir.yml index 1322965d..d7ec8140 100644 --- a/vendir.yml +++ b/vendir.yml @@ -6,7 +6,7 @@ directories: - path: cilium git: url: https://github.com/giantswarm/cilium - ref: "main" + ref: "v1.15" includePaths: - install/kubernetes/cilium/**/* - path: helm/cilium