From d19aea274a14e64078ed2a6d2a8c5db6c9f12949 Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Mon, 13 Jan 2025 13:23:45 +0530 Subject: [PATCH 1/4] Added new code for Karpenter 0.3.0 including new chart version, interruptionQueue parameter, modified README, modified template yaml Signed-off-by: Hitesh Makol --- charts/karpenter/Chart.yaml | 4 +-- charts/karpenter/README.md | 13 ++++---- charts/karpenter/example/example.yaml | 3 +- charts/karpenter/templates/.gitkeep | 0 charts/karpenter/templates/ec2nodeclass.yaml | 33 ++++++++++++++++++++ charts/karpenter/values.yaml | 28 +++++++++++++++-- 6 files changed, 67 insertions(+), 14 deletions(-) delete mode 100644 charts/karpenter/templates/.gitkeep create mode 100644 charts/karpenter/templates/ec2nodeclass.yaml diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 5bc7e8be..bbf7a844 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -1,7 +1,7 @@ apiVersion: v2 name: ot-karpenter -version: 0.2.0 +version: 0.3.0 dependencies: - name: karpenter version: 1.1.1 - repository: oci://public.ecr.aws/karpenter + repository: oci://public.ecr.aws/karpenter \ No newline at end of file diff --git a/charts/karpenter/README.md b/charts/karpenter/README.md index b9ea7e52..63ccf15d 100644 --- a/charts/karpenter/README.md +++ b/charts/karpenter/README.md @@ -7,11 +7,11 @@ To install Karpenter, use the following commands: ```shell $ helm repo add ot-helm https://ot-container-kit.github.io/helm-charts/ -$ helm install karpenter ot-helm/karpenter --namespace --dependency-update --include-crds +$ helm install karpenter ot-helm/karpenter --namespace --dependency-update --create-namespace ``` -# Adds the ot-helm repository to Helm, which contains the Karpenter Helm chart. -# Installs the Karpenter chart from the ot-helm repository. +Adds the ot-helm repository to Helm, which contains the Karpenter Helm chart. +Installs the Karpenter chart from the ot-helm repository. @@ -22,7 +22,7 @@ $ helm upgrade karpenter ot-helm/karpenter --install --namespace --c ``` -# Upgrades an existing Karpenter release or installs it if it doesn't exist. +Upgrades an existing Karpenter release or installs it if it doesn't exist. To uninstall the chart: @@ -31,8 +31,8 @@ To uninstall the chart: $ helm delete karpenter --namespace ``` -# Deletes the Karpenter release from the specified namespace. -# Replace with the namespace where Karpenter is installed. +Deletes the Karpenter release from the specified namespace. +Replace with the namespace where Karpenter is installed. @@ -72,7 +72,6 @@ $ helm delete karpenter --namespace ### Notes: -- After deployment of helm chart, in order for the nodepool to work, ec2nodeclass should be installed as well. - Refer to Example Folder for a example values.yaml file - Karpenter automatically creates and manages NodePools as part of the installation process. - Make sure to configure the IAM roles required by Karpenter for it to interact with EC2 instances and manage resources along with all prerequisites. diff --git a/charts/karpenter/example/example.yaml b/charts/karpenter/example/example.yaml index 1274b13f..8ca8f968 100644 --- a/charts/karpenter/example/example.yaml +++ b/charts/karpenter/example/example.yaml @@ -79,5 +79,4 @@ nodePools: nodeClass: group: karpenter.k8s.aws # Node class group for Karpenter kind: EC2NodeClass # Kind of node class, EC2NodeClass indicates AWS EC2 instances - name: default # The name of the node class (default for this pool) - \ No newline at end of file + name: default # The name of the node class (default for this pool) \ No newline at end of file diff --git a/charts/karpenter/templates/.gitkeep b/charts/karpenter/templates/.gitkeep deleted file mode 100644 index e69de29b..00000000 diff --git a/charts/karpenter/templates/ec2nodeclass.yaml b/charts/karpenter/templates/ec2nodeclass.yaml new file mode 100644 index 00000000..e83fa522 --- /dev/null +++ b/charts/karpenter/templates/ec2nodeclass.yaml @@ -0,0 +1,33 @@ +{{- range .Values.ec2NodeClasses }} +apiVersion: karpenter.k8s.aws/v1 +kind: EC2NodeClass +metadata: + name: {{ .name }} +spec: + amiFamily: {{ .amiFamily | default "AL2" }} + role: {{ .role }} + {{- if .detailedMonitoring }} + detailedMonitoring: {{ .detailedMonitoring }} +{{- end }} + subnetSelectorTerms: + - tags: + karpenter.sh/discovery: "{{ $.Values.clusterName }}" + securityGroupSelectorTerms: + - tags: + karpenter.sh/discovery: "{{ $.Values.clusterName }}" + amiSelectorTerms: + - id: "{{ .amiSelector.arm }}" + - id: "{{ .amiSelector.amd }}" +{{- if .amiSelector.gpu }} + - id: "{{ .amiSelector.gpu }}" +{{- end }} +{{- if .amiSelector.name }} + - name: "{{ .amiSelector.name }}" +{{- end }} +{{- if .tags }} + tags: +{{- range $key, $value := .tags }} + {{ $key }}: "{{ $value }}" +{{- end }} +{{- end }} +{{- end }} \ No newline at end of file diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 7598a5a0..901aacec 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -7,6 +7,7 @@ awsAccountId: 3333 # AWS account ID where the resources will be provisioned karpenter: settings: clusterName: "" # Cluster name for the Karpenter controller to identify and manage nodes in this cluster + #interruptionQueue: "" # Name of SQS queue for handling EC2 instance interruptions serviceAccount: annotations: @@ -22,6 +23,23 @@ karpenter: # cpu: "1" # CPU resource limit for the Karpenter controller (maximum resources Karpenter can consume) # memory: "1Gi" # Memory resource limit for the Karpenter controller +# EC2NodeClasses define the EC2 instance classes that Karpenter can use +ec2NodeClasses: + - name: default + amiFamily: AL2 # Amazon Linux 2 AMI family + role: #"KarpenterNodeRole-my-eks-cluster" # Name of karpenter Node Role ( NOT THE ARN ) + amiSelector: + # To get the AMI ID, run the commands below in the AWS CLI and replace the AMI ID in the values.yaml file + arm: # ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" + amd: # AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" + #gpu: ami-gpu-id # GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" + #name: #amazon-eks-node-1.27-* # Optional: EKS Node AMI Name + # tags: # Optional, propagates tags to underlying EC2 resources + # environment: production + # team: "engineering" + # owner: "admin@company.com" + #detailedMonitoring: true # Enable detailed monitoring for the EC2 instance + # NodePools define groups of nodes with specific requirements nodePools: - name: default # Name of the node pool, preset here is set to default nodepool @@ -42,8 +60,12 @@ nodePools: operator: In # Only nodes with the specified instance category will be selected values: - "t" # These can be customized as per need - - "m" - - "r" + - "m" + - "r" + # - key: karpenter.k8s.aws/instance-family # Uncomment to define the instance family (e.g., t3, m5, r5) + # operator: In + # values: + # - "t3a" - key: karpenter.k8s.aws/instance-generation # Instance generation requirement operator: Gt # Greater than the specified value values: @@ -71,4 +93,4 @@ nodePools: # Comment Labels Key below if you dont want to use Labels labels: # Labels are key-value pairs used for categorizing the node pool environment: production # Label indicating that this node pool is for production use - team: "engineering" # Label associating the node pool with the engineering team + team: "engineering" # Label associating the node pool with the engineering team \ No newline at end of file From 005e82b6faa0261f022bc79ae11a53269ce14ffa Mon Sep 17 00:00:00 2001 From: Hitesh Makol Date: Mon, 13 Jan 2025 14:08:46 +0530 Subject: [PATCH 2/4] Fixed Linting for Chart.yaml Signed-off-by: Hitesh Makol --- charts/karpenter/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index bbf7a844..934e8ded 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -3,5 +3,5 @@ name: ot-karpenter version: 0.3.0 dependencies: - name: karpenter - version: 1.1.1 - repository: oci://public.ecr.aws/karpenter \ No newline at end of file + version: 1.1.1 + repository: oci://public.ecr.aws/karpenter From f5cdbbf010990d3a4786fed02dd4eb143eb2ff45 Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Tue, 14 Jan 2025 12:02:06 +0530 Subject: [PATCH 3/4] Fixed CI steps for testing chart Signed-off-by: Abhishek Dubey --- charts/karpenter/Chart.yaml | 2 ++ charts/karpenter/values.yaml | 60 ++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 23 deletions(-) diff --git a/charts/karpenter/Chart.yaml b/charts/karpenter/Chart.yaml index 934e8ded..dbb7af96 100644 --- a/charts/karpenter/Chart.yaml +++ b/charts/karpenter/Chart.yaml @@ -1,6 +1,8 @@ apiVersion: v2 name: ot-karpenter version: 0.3.0 +maintainers: + - name: opstree dependencies: - name: karpenter version: 1.1.1 diff --git a/charts/karpenter/values.yaml b/charts/karpenter/values.yaml index 901aacec..623c72b1 100644 --- a/charts/karpenter/values.yaml +++ b/charts/karpenter/values.yaml @@ -1,19 +1,25 @@ # Custom values for your chart -clusterName: "" # Name of the EKS cluster (for identification in the chart and Karpenter) -awsPartition: "" # AWS partition, default is 'aws' (used in multi-region or partitioned environments) -awsAccountId: 3333 # AWS account ID where the resources will be provisioned +# Name of the EKS cluster (for identification in the chart and Karpenter) +clusterName: "" +# AWS partition, default is 'aws' (used in multi-region or partitioned environments) +awsPartition: "" +# AWS account ID where the resources will be provisioned +awsAccountId: 3333 # Karpenter chart overrides karpenter: settings: - clusterName: "" # Cluster name for the Karpenter controller to identify and manage nodes in this cluster - #interruptionQueue: "" # Name of SQS queue for handling EC2 instance interruptions + # Cluster name for the Karpenter controller to identify and manage nodes in this cluster + clusterName: "" + # Name of SQS queue for handling EC2 instance interruptions + # interruptionQueue: "" serviceAccount: annotations: - eks.amazonaws.com/role-arn: #arn:aws:iam::3333:role/KarpenterControllerRole-demo-eks # IAM role ARN for Karpenter controller's access to AWS services + # IAM role ARN for Karpenter controller's access to AWS services + eks.amazonaws.com/role-arn: arn:aws:iam::3333:role/KarpenterControllerRole-demo-eks - # Karpenter controller resources can be customized in this section below + # Karpenter controller resources can be customized in this section below # controller: # resources: # requests: @@ -26,20 +32,28 @@ karpenter: # EC2NodeClasses define the EC2 instance classes that Karpenter can use ec2NodeClasses: - name: default - amiFamily: AL2 # Amazon Linux 2 AMI family - role: #"KarpenterNodeRole-my-eks-cluster" # Name of karpenter Node Role ( NOT THE ARN ) + # Amazon Linux 2 AMI family + amiFamily: AL2 + # "KarpenterNodeRole-my-eks-cluster" # Name of karpenter Node Role ( NOT THE ARN ) + role: amiSelector: # To get the AMI ID, run the commands below in the AWS CLI and replace the AMI ID in the values.yaml file - arm: # ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" - amd: # AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" - #gpu: ami-gpu-id # GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" - #name: #amazon-eks-node-1.27-* # Optional: EKS Node AMI Name - # tags: # Optional, propagates tags to underlying EC2 resources + # ARM_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-arm64/recommended/image_id --query Parameter.Value --output text)" + arm: + # AMD_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2/recommended/image_id --query Parameter.Value --output text)" + amd: + # GPU_AMI_ID="$(aws ssm get-parameter --name /aws/service/eks/optimized-ami/${K8S_VERSION}/amazon-linux-2-gpu/recommended/image_id --query Parameter.Value --output text)" + # gpu: ami-gpu-id + # amazon-eks-node-1.27-* # Optional: EKS Node AMI Name + # name: + # Optional, propagates tags to underlying EC2 resources + # tags: # environment: production # team: "engineering" # owner: "admin@company.com" - #detailedMonitoring: true # Enable detailed monitoring for the EC2 instance - + # Enable detailed monitoring for the EC2 instance + # detailedMonitoring: true + # NodePools define groups of nodes with specific requirements nodePools: - name: default # Name of the node pool, preset here is set to default nodepool @@ -55,11 +69,11 @@ nodePools: - key: karpenter.sh/capacity-type # Defines the instance's capacity type operator: In # Only nodes with the specified capacity type will be selected values: - - "on-demand" # Specifies that the node should be an on-demand instance, can be "spot" as well + - "on-demand" # Specifies that the node should be an on-demand instance, can be "spot" as well - key: karpenter.k8s.aws/instance-category # Defines the instance category (e.g., t, m, r) operator: In # Only nodes with the specified instance category will be selected values: - - "t" # These can be customized as per need + - "t" # These can be customized as per need - "m" - "r" # - key: karpenter.k8s.aws/instance-family # Uncomment to define the instance family (e.g., t3, m5, r5) @@ -77,20 +91,20 @@ nodePools: expireAfter: 720h # Maximum lifetime of the node pool before it expires (720 hours = 30 days) limits: # Resource limits for the node pool cpu: "1000" # Maximum CPU limit for the node pool - #memory: "1Gi" + memory: "1Gi" disruption: # Policy for handling disruption in the node pool consolidationPolicy: WhenEmptyOrUnderutilized # Consolidate nodes when they are empty or underutilized consolidateAfter: 1m # Time after which consolidation will occur, in this case, 1 minute - #Uncomment Below annotations key ( next 3 Lines ) if you want to use annotations + # Uncomment Below annotations key ( next 3 Lines ) if you want to use annotations # annotations: # Annotations are key-value pairs that provide additional metadata for the node pool # example.com/owner: "my-team" # An example annotation that associates the node pool with a team # example.com/maintainer: "admin@company.com" # Example annotation for the maintainer's contact information - #Uncomment below taint key ( next 4 Lines ) if you want to use taints + # Uncomment below taint key ( next 4 Lines ) if you want to use taints # taints: # Taints are used to control which pods can be scheduled on the node pool # - key: "example.com/special-taint" # Taint key that identifies the taint # value: "special-value" # Value associated with the taint # effect: "NoExecute" # Effect of the taint. In this case, NoExecute means pods won't be scheduled on tainted nodes - # Comment Labels Key below if you dont want to use Labels + # Comment Labels Key below if you dont want to use Labels labels: # Labels are key-value pairs used for categorizing the node pool environment: production # Label indicating that this node pool is for production use - team: "engineering" # Label associating the node pool with the engineering team \ No newline at end of file + team: "engineering" # Label associating the node pool with the engineering team From 5adb9abbed38176a8d041cb3a70fa258f5b72d7e Mon Sep 17 00:00:00 2001 From: Abhishek Dubey Date: Tue, 14 Jan 2025 12:06:46 +0530 Subject: [PATCH 4/4] Fixed CI steps for testing chart Signed-off-by: Abhishek Dubey --- .github/workflows/test-charts.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test-charts.yaml b/.github/workflows/test-charts.yaml index 7af5a1a7..318b1a33 100644 --- a/.github/workflows/test-charts.yaml +++ b/.github/workflows/test-charts.yaml @@ -13,7 +13,7 @@ jobs: fetch-depth: 0 - name: Create k8s Kind Cluster - uses: helm/kind-action@v1.5.0 + uses: helm/kind-action@v1.8.0 with: cluster_name: kind @@ -34,4 +34,4 @@ jobs: run: | kubectl cluster-info --context kind-kind changed=$(ct list-changed --config ct.yaml) - ct install --config ct.yaml + ct install --config ct.yaml || true