feat(t8s-cluster/api-server): add support for OIDC

charts/t8s-cluster/templates/NOTES.txt

@@ -2,4 +2,22 @@ Use the following command to get the admin kubeconfig;
 
 ```
 $ kubectl -n {{ .Release.Namespace }} get secret "{{ .Release.Name }}-kubeconfig" -o jsonpath={.data.value} | base64 -d
-```
+```
+
+For OIDC, use the following kubeconfig user snippet:
+
+```yaml
+users:
+  - name: oidc
+    user:
+      exec:
+        apiVersion: client.authentication.k8s.io/v1beta1
+        command: kubectl
+        args:
+          - oidc-login
+          - get-token
+          - --oidc-issuer-url=${OIDC_ISSUER_URL?}
+          - --oidc-client-id=kubernetes
+          - --oidc-client-secret=${OIDC_CLIENT_SECRET?}
+          - --oidc-extra-scope=email
+```

charts/t8s-cluster/templates/management-cluster/clusterClass/_authenticationConfig.yaml

@@ -0,0 +1,15 @@
+{{- define "t8s-cluster.clusterClass.apiServer.authenticationConfig" -}}
+apiVersion: apiserver.config.k8s.io/v1beta1
+kind: AuthenticationConfiguration
+jwt:
+  - issuer:
+      discoveryURL: https://auth.k8s.teuto.net/.well-known/openid-configuration
+      audiences:
+        - kubernetes
+    claimMappings:
+      username:
+        expression: claims.email
+    claimValidationRules:
+      - expression: (has(claims.email_verified) && claims.email_verified) || !has(claims.email_verified)
+        message: email must be verified
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/_helpers.tpl

@@ -155,18 +155,70 @@ server = {{ printf "https://%s" .registry | quote }}
 {{- end }}
 
 {{- define "t8s-cluster.clusterClass.apiServer.admissionPlugins" -}}
-  {{- $admissionPlugins := list "AlwaysPullImages" "NodeRestriction" -}}
-  {{- if not .excludePatches -}}
-    {{- $admissionPlugins = concat $admissionPlugins (list "EventRateLimit") -}}
-  {{- end -}}
+  {{- $admissionPlugins := list "AlwaysPullImages" "NodeRestriction" "EventRateLimit" -}}
   {{- toYaml $admissionPlugins -}}
 {{- end -}}
 
+{{- define "t8s-cluster.clusterClass.configPath" -}}
+/etc/kubernetes
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.authenticationConfigPath" -}}
+{{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.authenticationConfigFileName" (dict) -}}
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" -}}
+{{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict) -}}
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" -}}
+{{- include "t8s-cluster.clusterClass.configPath" (dict) -}}/{{- include "t8s-cluster.clusterClass.apiServer.admissionControlConfigFileName" (dict) -}}
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.authenticationConfigFileName" -}}
+authentication-config.yaml
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" -}}
+event-rate-limit-config.yaml
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.admissionControlConfigFileName" -}}
+admission-control-config.yaml
+{{- end -}}
+
 {{- define "t8s-cluster.clusterClass.args.apiServer" -}}
   {{- $args := include "t8s-cluster.clusterClass.args.base" (dict "context" .context) | fromYaml -}}
   {{- $args = mustMerge (include "t8s-cluster.clusterClass.args.sharedController" (dict "context" .context) | fromYaml) $args -}}
-  {{- $args = set $args "enable-admission-plugins" (include "t8s-cluster.clusterClass.apiServer.admissionPlugins" (dict "excludePatches" .excludePatches) | fromYamlArray | join ",") -}}
+  {{- $args = set $args "authentication-config" (include "t8s-cluster.clusterClass.apiServer.authenticationConfigPath" (dict)) -}}
+  {{- $args = set $args "admission-control-config-file" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict)) -}}
+  {{- $args = set $args "enable-admission-plugins" (include "t8s-cluster.clusterClass.apiServer.admissionPlugins" (dict) | fromYamlArray | join ",") -}}
   {{- $args = set $args "event-ttl" "4h" -}}
   {{- $args = set $args "tls-cipher-suites" (include "t8s-cluster.clusterClass.tlsCipherSuites" (dict) | fromYamlArray | join ",") -}}
   {{- toYaml $args -}}
 {{- end }}
+
+{{- define "t8s-cluster.clusterClass.apiServer.staticFiles" -}}
+  {{- toYaml (dict
+      "admission-control-config.yaml" (dict
+        "path" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict))
+        "fileName" (include "t8s-cluster.clusterClass.apiServer.admissionControlConfigFileName" (dict))
+      )
+      "event-rate-limit-config.yaml" (dict
+        "path" (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" (dict))
+        "fileName" (include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigFileName" (dict))
+      )
+    )
+  -}}
+{{- end -}}
+
+{{- define "t8s-cluster.clusterClass.apiServer.dynamicFiles" -}}
+  {{- toYaml (dict
+      "authentication-config.yaml" (dict
+        "path" (include "t8s-cluster.clusterClass.apiServer.authenticationConfigPath" (dict))
+        "fileName" (include "t8s-cluster.clusterClass.apiServer.authenticationConfigFileName" (dict))
+        "content" (include "t8s-cluster.clusterClass.apiServer.authenticationConfig" (dict "context" .))
+      )
+    )
+  -}}
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/clusterClass.yaml

@@ -87,7 +87,7 @@ spec:
       definitions:
         - jsonPatches:
             - &imagePatch
-              op: add
+              op: replace
               path: /spec/template/spec/image/filter/name
               valueFrom:
                 template: {{ printf "t8s-engine-%s{{ .builtin.machineDeployment.version }}" .Values.openstackImageNamePrefix }}

charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/_k0smotronControlPlaneTemplateSpec.yaml

@@ -11,25 +11,23 @@ service:
   konnectivityPort: 8132
 controllerPlaneFlags:
   - --disable-components=metrics-server
+mounts:
+  - configMap:
+      name: {{ printf "%s-apiserver" $.Release.Name }}
+      {{- $items := list -}}
+      {{- range $name, $file := mustMerge (include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml) (include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml) -}}
+        {{- $items = append $items (dict "key" $name "path" (get $file "fileName" | required (printf "missing fileName for %s" $file))) -}}
+      {{- end }}
+      items: {{- toYaml $items | nindent 6 }}
+    path: {{ include "t8s-cluster.clusterClass.configPath" (dict) }}
 k0sConfig:
   apiVersion: k0s.k0sproject.io/v1beta1
   kind: ClusterConfig
   spec:
     network:
       provider: custom
     api:
-      extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" . "excludePatches" true) | nindent 8 }}
-        # TODO: figure out how to make this work
-        # admission-control-config-file: &admissionControlConfigFilePath /etc/kubernetes/admission-control-config.yaml
-        #  extraVolumes:
-        #    - hostPath: *admissionControlConfigFilePath
-        #      mountPath: *admissionControlConfigFilePath
-        #      name: admission-control-config
-        #      readOnly: true
-        #    - hostPath: &eventRateLimitConfigFilePath {\{ $eventRateLimitConfigFilePath }}
-        #      mountPath: *eventRateLimitConfigFilePath
-        #      name: event-rate-limit-config
-        #      readOnly: true
+      extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 8 }}
     controllerManager:
       extraArgs: {{- include "t8s-cluster.clusterClass.args.controllerManager" (dict "context" .) | nindent 8 }}
     scheduler:

charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneConfigMap.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.controlPlane.hosted -}}
+  {{- $data := dict -}}
+  {{- range $name := include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml | keys -}}
+    {{- $data = set $data $name ($.Files.Get (printf "files/%s" $name)) -}}
+  {{- end -}}
+  {{- range $name, $file := include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml -}}
+    {{- $data = set $data $name (get $file "content" | required (printf "missing content for %s" $name)) -}}
+  {{- end -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ printf "%s-apiserver" $.Release.Name }}
+  namespace: {{ $.Release.Namespace }}
+  labels: {{- include "common.labels.standard" $ | nindent 4 }}
+data: {{- toYaml $data | nindent 2 -}}
+{{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneTemplate.yaml

@@ -7,5 +7,8 @@ metadata:
   labels: {{- include "common.labels.standard" $ | nindent 4 }}
 spec:
   template:
+    metadata:
+      annotations:
+        checksum/configmap: {{ include "common.utils.checksumTemplate" (dict "path" "/management-cluster/clusterClass/k0smotronControlPlaneTemplate/k0smotronControlPlaneConfigMap.yaml" "context" $) }}
     spec: {{/* the full context is needed for .Files.Get */}}{{- include "t8s-cluster.clusterClass.k0smotronControlPlaneTemplate.spec" . | nindent 6 }}
 {{- end -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_helpers.tpl

@@ -17,18 +17,20 @@
 {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.files" -}}
   {{- $files := list -}}
   {{- $files = concat $files (include "t8s-cluster.clusterClass.configTemplate.files" (dict "context" . "gpu" false) | fromYamlArray) -}}
-  {{- $configs := dict
-    "admission-control-config.yaml" (required "Missing" .admissionControlConfigFilePath)
-    "event-rate-limit-config.yaml" (required "Missing" .eventRateLimitConfigFilePath)
-    "kube-proxy.config.yaml" "/etc/kube-proxy-config.yaml"
-  -}}
-  {{- range $file, $path := $configs -}}
-    {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $file)) "path" $path) -}}
+  {{- $configs := include "t8s-cluster.clusterClass.apiServer.staticFiles" (dict) | fromYaml -}}
+  {{- $configs = mustMerge $configs (dict "kube-proxy.config.yaml" (dict "path" "/etc/kube-proxy-config.yaml")) -}}
+  {{- range $name, $file := $configs -}}
+    {{- $files = append $files (dict "content" ($.Files.Get (printf "files/%s" $name)) "path" (get $file "path" | required (printf "missing path for %s" $name))) -}}
   {{- end -}}
   {{- $files = append $files (dict "content" (.Files.Get "files/kube-proxy.patch.sh") "path" "/etc/kube-proxy-patch.sh" "permissions" "0700") -}}
   {{- range $file := $files -}}
-    {{- $_ := set $file "content" (get $file "content" | trim) -}}
+    {{- $file = set $file "content" (get $file "content" | trim) -}}
   {{- end -}}
+  {{- $dynamicFiles := include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml | values -}}
+  {{- range $name, $file := $dynamicFiles -}}
+    {{- $files = append $files (dict "content" (get $file "content" | required (printf "missing content for %s" $name)) "path" (get $file "path" | required (printf "missing path for %s" $name))) -}}
+  {{- end -}}
+  {{- $files = concat $files (include "t8s-cluster.clusterClass.apiServer.dynamicFiles" (dict "context" .) | fromYaml | values) -}}
   {{- $apiserverPatch := dict "spec" (dict "containers" (list (dict "name" "kube-apiserver" "resources" (dict "requests" (dict "memory" "2Gi") "limits" (dict "memory" "4Gi"))))) -}}
   {{- $files = append $files (include "t8s-cluster.patches.patchFile" (dict "values" $apiserverPatch "target" "kube-apiserver" "component" "memory") | fromYaml) -}}
   {{- toYaml $files -}}

charts/t8s-cluster/templates/management-cluster/clusterClass/kubeadmControlPlaneTemplate/_kubeadmControlPlaneTemplateSpec.yaml

@@ -4,21 +4,18 @@ Here we are generating a hash suffix.
 This function needs the whole `$` context to be able to use `.Files.Get`
 */}}
 {{- define "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.spec" -}}
-  {{- $admissionControlConfigFilePath := "/etc/kubernetes/admission-control-config.yaml" -}}
-  {{- $eventRateLimitConfigFilePath := "/etc/kubernetes/event-rate-limit-config.yaml" -}}
 rolloutBefore:
   certificatesExpiryDays: 60
 kubeadmConfigSpec:
   clusterConfiguration:
     apiServer:
       extraArgs: {{- include "t8s-cluster.clusterClass.args.apiServer" (dict "context" .) | nindent 8 }}
-        admission-control-config-file: &admissionControlConfigFilePath {{ $admissionControlConfigFilePath }}
       extraVolumes:
-        - hostPath: *admissionControlConfigFilePath
+        - hostPath: &admissionControlConfigFilePath {{ include "t8s-cluster.clusterClass.apiServer.admissionControlConfigPath" (dict) }}
           mountPath: *admissionControlConfigFilePath
           name: admission-control-config
           readOnly: true
-        - hostPath: &eventRateLimitConfigFilePath {{ $eventRateLimitConfigFilePath }}
+        - hostPath: &eventRateLimitConfigFilePath {{ include "t8s-cluster.clusterClass.apiServer.eventRateLimitConfigPath" (dict) }}
           mountPath: *eventRateLimitConfigFilePath
           name: event-rate-limit-config
           readOnly: true
@@ -30,7 +27,7 @@ kubeadmConfigSpec:
           listen-metrics-urls: http://0.0.0.0:2381
     scheduler:
       extraArgs: {{- include "t8s-cluster.clusterClass.args.scheduler" (dict) | nindent 8 }}
-  files: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.files" (merge . (dict "admissionControlConfigFilePath" $admissionControlConfigFilePath "eventRateLimitConfigFilePath" $eventRateLimitConfigFilePath)) | nindent 4 }}
+  files: {{- include "t8s-cluster.clusterClass.kubeadmControlPlaneTemplate.files" . | nindent 4 }}
   initConfiguration: &configuration
     nodeRegistration:
       kubeletExtraArgs: {{- include "t8s-cluster.clusterClass.kubeletExtraArgs" (dict "context" .) | nindent 8 }}

charts/t8s-cluster/templates/management-cluster/clusterClass/openStackClusterTemplate/_openStackClusterTemplateSpec.yaml

@@ -51,7 +51,7 @@ identityRef:
   {{- range $name, $securityGroupRule := $cniSecurityGroupRules -}}
     {{- $_securityGroupRule := dict
       "name" (printf "%s %s" $cni $name)
-      "direction" "Ingress"
+      "direction" "ingress"
       "etherType" "IPv4"
       "protocol" ($securityGroupRule.protocol | required "security group rule protocol is required")
       "remoteManagedGroups" $remoteManagedGroups

charts/t8s-cluster/templates/workload-cluster/cloud-controller-manager.yaml

@@ -53,8 +53,13 @@ spec:
               - op: replace
                 path: /kind
                 value: Deployment
+              {{- if gt (.Values.version.minor | int) 28 }}
+              - op: remove
+                path: /spec/template/spec/nodeSelector
+              {{- end }}
   {{- end }}
   values:
+    priorityClassName: system-cluster-critical
   {{- if le (.Values.version.minor | int) 28 }}
     tolerations:
       - key: node.cloudprovider.kubernetes.io/uninitialized
@@ -66,8 +71,4 @@ spec:
     nodeSelector:
       node-role.kubernetes.io/control-plane: ""
     {{- end }}
-  {{- else }}
-    {{- if .Values.controlPlane.hosted }}
-    nodeSelector: null
-    {{- end }}
   {{- end }}