Merge master into release-1.13 for VPC CNI v1.13.3 release (#2464)

* refactor canary test to access images from AWS registries (#2398) * upgrade client-go and controller-runtime modules (#2396) * updates for v1.13.0 release (#2400) * chore: Added dependabot (#2403) * dependency updates (#2412) * deprecate ENABLE_NFTABLES and set iptables mode using iptables-wrapper script (#2402) * update networking test agent to go1.20 and latest sys module (#2413) * skip delete test cluster to debug (#2414) * Revert "skip delete test cluster to debug (#2414)" (#2415) This reverts commit 7c30943. * authenticate to test image registry (#2417) * update test agent image (#2419) * update test agent hash in go.mod (#2422) * fix hard-coded nitro instances (#2428) * move authentication step from test canary script (#2429) * node initialization must come after primary ENI's security groups are synced to cache (#2427) * Add 1.27 to Rec Version Table (#2404) * revise rec version table * make DOCKER_ARGS a passable var from CLI builds (#2434) Signed-off-by: jonahjon <[email protected]> * Update Kops cluster to latest and add parameter for kops version (#2435) * Updates instance limits including c7gn (#2438) * Update Kops cluster to latest and add parameter for kops version (#2440) * update image tag to v1.13.2 (#2432) * update docs and CNI logging (#2433) * remove default canary test run from integration tests (#2443) * Silences nightly cron jobs for forks (#2444) * Silences weekly cron jobs for forks (#2459) * refactor performance tests (#2455) * add custom-networking test covering ENIConfig objects with no security (#2445) groups * k8s clients only need to access corev1; add pod selector (#2463) --------- Signed-off-by: jonahjon <[email protected]> Co-authored-by: Olivia Song <[email protected]> Co-authored-by: Ellis Tarn <[email protected]> Co-authored-by: Geoffrey Cline <[email protected]> Co-authored-by: Jonah Jones <[email protected]> Co-authored-by: Jay Deokar <[email protected]> Co-authored-by: Matt <[email protected]> Co-authored-by: Matt <[email protected]>
aws · Jul 11, 2023 · a6e991f · a6e991f
1 parent c5f5c6e
commit a6e991f
Show file tree

Hide file tree

Showing 32 changed files with 325 additions and 357 deletions.
diff --git a/.github/workflows/integration-tests.yaml b/.github/workflows/integration-tests.yaml
@@ -23,7 +23,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Set up tools
         run: |
           # Install ginkgo version from go.mod
@@ -39,8 +39,6 @@ jobs:
       - name: Run e2e tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
           RUN_CONFORMANCE: true

diff --git a/.github/workflows/nightly-cron-tests.yaml b/.github/workflows/nightly-cron-tests.yaml
@@ -10,6 +10,7 @@ permissions:
 
 jobs:
   nightly-cron-test:
+    if: github.repository == 'aws/amazon-vpc-cni-k8s'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout latest commit in the PR
@@ -21,7 +22,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Set up tools
         run: |
           # Install ginkgo version from go.mod
@@ -37,8 +38,6 @@ jobs:
       - name: Run e2e tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
           RUN_CONFORMANCE: true

diff --git a/.github/workflows/pr-automated-tests.yaml b/.github/workflows/pr-automated-tests.yaml
@@ -19,7 +19,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Set up tools
         run: |
           go install golang.org/x/lint/golint@latest
@@ -49,7 +49,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Build CNI images
         run: make multi-arch-cni-build
       - name: Build CNI Init images

diff --git a/.github/workflows/pr-manual-tests.yaml b/.github/workflows/pr-manual-tests.yaml
@@ -29,7 +29,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Set up tools
         run: |
           # Install ginkgo version from go.mod
@@ -45,11 +45,8 @@ jobs:
       - name: Run e2e tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
           RUN_CONFORMANCE: true
-          RUN_INTEGRATION_DEFAULT_CNI: false
         run: |
           ./scripts/run-integration-tests.sh
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -22,7 +22,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Generate CNI YAML
         run: make generate-cni-yaml
       - name: Create eks-charts PR

diff --git a/.github/workflows/weekly-cron-tests.yaml b/.github/workflows/weekly-cron-tests.yaml
@@ -11,6 +11,7 @@ permissions:
 
 jobs:
   weekly-cron:
+    if: github.repository == 'aws/amazon-vpc-cni-k8s'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout latest commit in the PR
@@ -22,7 +23,7 @@ jobs:
       - name: Set up Go
         uses: actions/setup-go@v3
         with:
-          go-version: "1.19"
+          go-version: "1.20"
       - name: Set up tools
         run: |
           # Install ginkgo version from go.mod
@@ -38,54 +39,47 @@ jobs:
       - name: Run perf tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
+          RUN_CNI_INTEGRATION_TESTS: false
           PERFORMANCE_TEST_S3_BUCKET_NAME: cni-performance-tests
           RUN_PERFORMANCE_TESTS: true
           RUN_TESTER_LB_ADDONS: true
-          RUN_INTEGRATION_DEFAULT_CNI: false
         run: |
           ./scripts/run-integration-tests.sh
       - name: Run kops tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
+          RUN_CNI_INTEGRATION_TESTS: false
           RUN_KOPS_TEST: true
           RUN_TESTER_LB_ADDONS: true
-          K8S_VERSION: 1.26.0
-          RUN_INTEGRATION_DEFAULT_CNI: false
+          K8S_VERSION: 1.26.5
+          KOPS_VERSION: v1.26.4
         run: |
           ./scripts/run-integration-tests.sh
         if: always()
       - name: Run bottlerocket tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
+          RUN_CNI_INTEGRATION_TESTS: false
           RUN_BOTTLEROCKET_TEST: true
           RUN_TESTER_LB_ADDONS: true
-          RUN_INTEGRATION_DEFAULT_CNI: false
         run: |
           ./scripts/run-integration-tests.sh
         if: always()
       - name: Run calico tests
         env:
           DISABLE_PROMPT: true
-          S3_BUCKET_CREATE: false
-          S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }}
           ROLE_CREATE: false
           ROLE_ARN: ${{ secrets.EKS_CLUSTER_ROLE_ARN }}
+          RUN_CNI_INTEGRATION_TESTS: false
           RUN_CALICO_TEST: true
           RUN_LATEST_CALICO_VERSION: true
           RUN_TESTER_LB_ADDONS: true
-          RUN_INTEGRATION_DEFAULT_CNI: false
         run: |
           ./scripts/run-integration-tests.sh
         if: always()
diff --git a/Makefile b/Makefile
@@ -89,7 +89,7 @@ BINS = aws-k8s-agent aws-cni grpc-health-probe cni-metrics-helper aws-vpc-cni aw
 CORE_PLUGIN_DIR = $(MAKEFILE_PATH)/core-plugins/
 
 # DOCKER_ARGS is extra arguments passed during container image build.
-DOCKER_ARGS =
+DOCKER_ARGS ?=
 # DOCKER_RUN_FLAGS is set the flags passed during runs of containers.
 DOCKER_RUN_FLAGS = --rm -ti $(DOCKER_ARGS)
 # DOCKER_BUILD_FLAGS_CNI is the set of flags passed during CNI container image 

diff --git a/README.md b/README.md
@@ -67,9 +67,9 @@ For help, please consider the following venues (in order):
 For all Kubernetes releases, we recommend installing the latest VPC CNI release. The following table denotes our minimum recommended
 VPC CNI version for each actively supported Kubernetes release.
 
-| Kubernetes Release |    1.26    |   1.25    |    1.24    |    1.23    |    1.22    |
-| ------------------ | ---------- | ----------| ---------- | ---------- | ---------- |
-|  VPC CNI Version   |  v1.12.0+  | v1.11.4+  |  v1.9.3+   |  v1.8.0+   |  v1.8.0+   |
+| Kubernetes Release | 1.27     | 1.26     | 1.25     | 1.24    | 1.23    |
+| ------------------ | -------- | -------- | -------- | ------- | ------- |
+| VPC CNI Version    | v1.12.5+ | v1.12.0+ | v1.11.4+ | v1.9.3+ | v1.8.0+ |
 
 ## Version Upgrade
 
@@ -316,7 +316,7 @@ Default: `DEBUG`
 
 Valid Values: `DEBUG`, `INFO`, `WARN`, `ERROR`, `FATAL`. (Not case sensitive)
 
-Specifies the loglevel for `ipamd` and `cni-metric-helper`.
+Specifies the log level for `ipamd` and `cni-metric-helper`.
 
 ---
 
@@ -326,9 +326,13 @@ Type: String
 
 Default: `/host/var/log/aws-routed-eni/ipamd.log`
 
-Valid Values: `stdout` or a file path
+Valid Values: `stdout`, `stderr`, or a file path
 
-Specifies where to write the logging output of `ipamd`. Either to stdout or to override the default file (i.e., `/var/log/aws-routed-eni/ipamd.log`).
+Specifies where to write the logging output of `ipamd`: `stdout`, `stderr`, or a file path other than the default (`/var/log/aws-routed-eni/ipamd.log`).
+
+Note: `/host/var/log/...` is the container file-system path, which maps to `/var/log/...` on the node.
+
+Note: The IPAMD process runs within the `aws-node` pod, so writing to `stdout` or `stderr` will write to `aws-node` pod logs.
 
 ---
 
@@ -338,12 +342,15 @@ Type: String
 
 Default: `/var/log/aws-routed-eni/plugin.log`
 
-Valid Values: `stderr` or a file path
+Valid Values: `stderr` or a file path. Note that setting to the empty string is an alias for `stderr`, and this comes from upstream kubernetes best practices.
+
+Specifies where to write the logging output for `aws-cni` plugin: `stderr` or a file path other than the default (`/var/log/aws-routed-eni/plugin.log`).
+
+Note: `stdout` cannot be supported for plugin log. Please refer to [#1248](https://github.com/aws/amazon-vpc-cni-k8s/issues/1248) for more details.
 
-Specifies where to write the logging output for `aws-cni` plugin. Either to `stderr` or to override the default file (i.e., `/var/log/aws-routed-eni/plugin.log`).
-`Stdout` cannot be supported for plugin log, please refer to [#1248](https://github.com/aws/amazon-vpc-cni-k8s/issues/1248) for more details.
+Note: In EKS 1.24+, the CNI plugin is exec'ed by the container runtime, so `stderr` is for the container-runtime process, NOT the `aws-node` pod. In older versions, the CNI plugin was exec'ed by kubelet, so `stderr` is for the kubelet process.
 
-Note: If chaining an external plugin (i.e Cilium) that does not provide a `pluginLogFile` in its config file, the CNI plugin will by default write to `os.Stderr`. The output of `cmdAdd` are available in the Kubelet logs.
+Note: If chaining an external plugin (i.e. Cilium) that does not provide a `pluginLogFile` in its config file, the CNI plugin will by default write to `os.Stderr`.
 
 ---
 

diff --git a/cmd/aws-k8s-agent/main.go b/cmd/aws-k8s-agent/main.go
@@ -52,7 +52,7 @@ func _main() int {
 		return 1
 	}
 
-	cacheK8SClient, err := k8sapi.CreateCachedKubeClient(rawK8SClient, mapper)
+	cacheK8SClient, err := k8sapi.CreateCachedKubeClient(rawK8SClient, mapper, true)
 	if err != nil {
 		log.Errorf("Failed to create cached kube client: %s", err)
 		return 1

diff --git a/cmd/cni-metrics-helper/main.go b/cmd/cni-metrics-helper/main.go
@@ -118,7 +118,7 @@ func main() {
 		log.Fatalf("Error creating Kubernetes Client: %s", err)
 		os.Exit(1)
 	}
-	k8sClient, err := k8sapi.CreateCachedKubeClient(rawK8SClient, mapper)
+	k8sClient, err := k8sapi.CreateCachedKubeClient(rawK8SClient, mapper, false)
 	if err != nil {
 		log.Fatalf("Error creating Cached Kubernetes Client: %s", err)
 		os.Exit(1)

diff --git a/cmd/cni-metrics-helper/metrics/pod_watcher.go b/cmd/cni-metrics-helper/metrics/pod_watcher.go
@@ -56,6 +56,6 @@ func (d *defaultPodWatcher) GetCNIPods(ctx context.Context) ([]string, error) {
 		CNIPods = append(CNIPods, pod.Name)
 	}
 
-	d.log.Infof("Total aws-node pod count:- ", len(CNIPods))
+	d.log.Infof("Total aws-node pod count: %d", len(CNIPods))
 	return CNIPods, nil
 }
diff --git a/misc/eni-max-pods.txt b/misc/eni-max-pods.txt
@@ -176,6 +176,14 @@ c7g.large 29
 c7g.medium 8
 c7g.metal 737
 c7g.xlarge 58
+c7gn.12xlarge 234
+c7gn.16xlarge 737
+c7gn.2xlarge 58
+c7gn.4xlarge 234
+c7gn.8xlarge 234
+c7gn.large 29
+c7gn.medium 8
+c7gn.xlarge 58
 cr1.8xlarge 234
 d2.2xlarge 58
 d2.4xlarge 234
@@ -233,6 +241,9 @@ h1.4xlarge 234
 h1.8xlarge 234
 hpc6a.48xlarge 100
 hpc6id.32xlarge 51
+hpc7g.16xlarge 198
+hpc7g.4xlarge 198
+hpc7g.8xlarge 198
 hs1.8xlarge 234
 i2.2xlarge 58
 i2.4xlarge 234

diff --git a/pkg/awsutils/vpc_ip_resource_limit.go b/pkg/awsutils/vpc_ip_resource_limit.go
diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go
@@ -568,7 +568,7 @@ func (c *IPAMContext) nodeInit() error {
 
 	node, err := k8sapi.GetNode(ctx, c.cachedK8SClient)
 	if err != nil {
-		log.Errorf("Failed to host node", err)
+		log.Errorf("Failed to get node", err)
 		podENIErrInc("nodeInit")
 		return err
 	}

diff --git a/pkg/k8sapi/k8sutils.go b/pkg/k8sapi/k8sutils.go
@@ -8,8 +8,8 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
-	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
 	eniconfigscheme "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
@@ -19,7 +19,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
-	"sigs.k8s.io/controller-runtime/pkg/cache"
+	crcache "sigs.k8s.io/controller-runtime/pkg/cache"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -45,7 +45,7 @@ func CreateKubeClient(mapper meta.RESTMapper) (client.Client, error) {
 		return nil, err
 	}
 	vpcCniScheme := runtime.NewScheme()
-	clientgoscheme.AddToScheme(vpcCniScheme)
+	corev1.AddToScheme(vpcCniScheme)
 	eniconfigscheme.AddToScheme(vpcCniScheme)
 
 	rawK8SClient, err := client.New(restCfg, client.Options{Scheme: vpcCniScheme, Mapper: mapper})
@@ -58,19 +58,28 @@ func CreateKubeClient(mapper meta.RESTMapper) (client.Client, error) {
 }
 
 // CreateKubeClient creates a k8s client
-func CreateCachedKubeClient(rawK8SClient client.Client, mapper meta.RESTMapper) (client.Client, error) {
+func CreateCachedKubeClient(rawK8SClient client.Client, mapper meta.RESTMapper, limitPods bool) (client.Client, error) {
 	restCfg, err := getRestConfig()
 	if err != nil {
 		return nil, err
 	}
 	restCfg.Burst = 100
 
 	vpcCniScheme := runtime.NewScheme()
-	clientgoscheme.AddToScheme(vpcCniScheme)
+	// For the cached client, IPAMD only needs nodes and ENIConfigs to be cached. Nodes come from corev1.
+	corev1.AddToScheme(vpcCniScheme)
 	eniconfigscheme.AddToScheme(vpcCniScheme)
 
 	stopChan := ctrl.SetupSignalHandler()
-	cache, err := cache.New(restCfg, cache.Options{Scheme: vpcCniScheme, Mapper: mapper})
+	// IPAMD only needs to cache pods on this node, so the following selector is used to reduce memory consumption
+	cacheOptions := crcache.Options{Scheme: vpcCniScheme, Mapper: mapper}
+	if nodeName := os.Getenv("MY_NODE_NAME"); limitPods && nodeName != "" {
+		cacheOptions.SelectorsByObject = map[client.Object]crcache.ObjectSelector{&corev1.Pod{}: {
+			Field: fields.Set{"spec.nodeName": nodeName}.AsSelector(),
+		}}
+
+	}
+	cache, err := crcache.New(restCfg, cacheOptions)
 	if err != nil {
 		return nil, err
 	}