Merge pull request #3413 from kinarashah/extaws5

add logic for external aws cloud provider

cluster/addons.go

@@ -492,7 +492,7 @@ func (c *Cluster) doAddonDeploy(ctx context.Context, addonYaml, resourceName str
 	if err != nil {
 		return &addonError{fmt.Sprintf("%v", err), isCritical}
 	}
-	node, err := k8s.GetNode(k8sClient, c.ControlPlaneHosts[0].HostnameOverride)
+	node, err := k8s.GetNode(k8sClient, c.ControlPlaneHosts[0].HostnameOverride, c.CloudProvider.Name)
 	if err != nil {
 		return &addonError{fmt.Sprintf("Failed to get Node [%s]: %v", c.ControlPlaneHosts[0].HostnameOverride, err), isCritical}
 	}
@@ -513,7 +513,7 @@ func (c *Cluster) doAddonDelete(ctx context.Context, resourceName string, isCrit
 	if err != nil {
 		return &addonError{fmt.Sprintf("%v", err), isCritical}
 	}
-	node, err := k8s.GetNode(k8sClient, c.ControlPlaneHosts[0].HostnameOverride)
+	node, err := k8s.GetNode(k8sClient, c.ControlPlaneHosts[0].HostnameOverride, c.CloudProvider.Name)
 	if err != nil {
 		return &addonError{fmt.Sprintf("Failed to get Node [%s]: %v", c.ControlPlaneHosts[0].HostnameOverride, err), isCritical}
 	}

cluster/cluster.go

@@ -189,7 +189,7 @@ func (c *Cluster) UpgradeControlPlane(ctx context.Context, kubeClient *kubernete
 			continue
 		}
 		// find existing nodes that are in NotReady state
-		if err := services.CheckNodeReady(kubeClient, host, services.ControlRole); err != nil {
+		if err := services.CheckNodeReady(kubeClient, host, services.ControlRole, c.CloudProvider.Name); err != nil {
 			logrus.Debugf("Found node %v in NotReady state", host.HostnameOverride)
 			notReadyHosts = append(notReadyHosts, host)
 			notReadyHostNames = append(notReadyHostNames, host.HostnameOverride)
@@ -223,7 +223,7 @@ func (c *Cluster) UpgradeControlPlane(ctx context.Context, kubeClient *kubernete
 		}
 		// Calling CheckNodeReady wil give some time for nodes to get in Ready state
 		for _, host := range notReadyHosts {
-			err = services.CheckNodeReady(kubeClient, host, services.ControlRole)
+			err = services.CheckNodeReady(kubeClient, host, services.ControlRole, c.CloudProvider.Name)
 			if err != nil {
 				logrus.Errorf("Host %v failed to report Ready status with error: %v", host.HostnameOverride, err)
 			}
@@ -236,7 +236,8 @@ func (c *Cluster) UpgradeControlPlane(ctx context.Context, kubeClient *kubernete
 		cpNodePlanMap,
 		c.UpdateWorkersOnly,
 		c.SystemImages.Alpine,
-		c.Certificates, c.UpgradeStrategy, c.NewHosts, inactiveHosts, c.MaxUnavailableForControlNodes, c.Version)
+		c.Certificates, c.UpgradeStrategy, c.NewHosts, inactiveHosts, c.MaxUnavailableForControlNodes,
+		c.Version, c.CloudProvider.Name)
 	if err != nil {
 		return "", fmt.Errorf("[controlPlane] Failed to upgrade Control Plane: %v", err)
 	}
@@ -310,7 +311,7 @@ func (c *Cluster) UpgradeWorkerPlane(ctx context.Context, kubeClient *kubernetes
 			continue
 		}
 		// find existing nodes that are in NotReady state
-		if err := services.CheckNodeReady(kubeClient, host, services.WorkerRole); err != nil {
+		if err := services.CheckNodeReady(kubeClient, host, services.WorkerRole, c.CloudProvider.Name); err != nil {
 			logrus.Debugf("Found node %v in NotReady state", host.HostnameOverride)
 			notReadyHosts = append(notReadyHosts, host)
 			notReadyHostNames = append(notReadyHostNames, host.HostnameOverride)
@@ -332,7 +333,7 @@ func (c *Cluster) UpgradeWorkerPlane(ctx context.Context, kubeClient *kubernetes
 		}
 		// Calling CheckNodeReady wil give some time for nodes to get in Ready state
 		for _, host := range notReadyHosts {
-			err = services.CheckNodeReady(kubeClient, host, services.WorkerRole)
+			err = services.CheckNodeReady(kubeClient, host, services.WorkerRole, c.CloudProvider.Name)
 			if err != nil {
 				logrus.Errorf("Host %v failed to report Ready status with error: %v", host.HostnameOverride, err)
 			}
@@ -349,7 +350,8 @@ func (c *Cluster) UpgradeWorkerPlane(ctx context.Context, kubeClient *kubernetes
 		c.UpgradeStrategy,
 		c.NewHosts,
 		c.MaxUnavailableForWorkerNodes,
-		c.Version)
+		c.Version,
+		c.CloudProvider.Name)
 	if err != nil {
 		return "", fmt.Errorf("[workerPlane] Failed to upgrade Worker Plane: %v", err)
 	}
@@ -994,7 +996,7 @@ func (c *Cluster) SyncLabelsAndTaints(ctx context.Context, currentCluster *Clust
 				var errs []error
 				for host := range hostQueue {
 					logrus.Debugf("worker [%d] starting sync for node [%s]", w, host.HostnameOverride)
-					if err := setNodeAnnotationsLabelsTaints(k8sClient, host); err != nil {
+					if err := setNodeAnnotationsLabelsTaints(k8sClient, host, c.CloudProvider.Name); err != nil {
 						errs = append(errs, err)
 					}
 				}
@@ -1012,11 +1014,11 @@ func (c *Cluster) SyncLabelsAndTaints(ctx context.Context, currentCluster *Clust
 	return nil
 }
 
-func setNodeAnnotationsLabelsTaints(k8sClient *kubernetes.Clientset, host *hosts.Host) error {
+func setNodeAnnotationsLabelsTaints(k8sClient *kubernetes.Clientset, host *hosts.Host, cloudProviderName string) error {
 	node := &v1.Node{}
 	var err error
 	for retries := 0; retries <= 5; retries++ {
-		node, err = k8s.GetNode(k8sClient, host.HostnameOverride)
+		node, err = k8s.GetNode(k8sClient, host.HostnameOverride, cloudProviderName)
 		if err != nil {
 			logrus.Debugf("[hosts] Can't find node by name [%s], error: %v", host.HostnameOverride, err)
 			time.Sleep(2 * time.Second)

cluster/defaults.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/rancher/rke/cloudprovider"
+	"github.com/rancher/rke/cloudprovider/aws"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/k8s"
 	"github.com/rancher/rke/log"
@@ -1022,11 +1023,25 @@ func (c *Cluster) setCloudProvider() error {
 	if p != nil {
 		c.CloudConfigFile, err = p.GenerateCloudConfigFile()
 		if err != nil {
-			return fmt.Errorf("Failed to parse cloud config file: %v", err)
+			return fmt.Errorf("failed to parse cloud config file: %v", err)
 		}
 		c.CloudProvider.Name = p.GetName()
 		if c.CloudProvider.Name == "" {
-			return fmt.Errorf("Name of the cloud provider is not defined for custom provider")
+			return fmt.Errorf("name of the cloud provider is not defined for custom provider")
+		}
+		if c.CloudProvider.Name == aws.AWSCloudProviderName {
+			clusterVersion, err := getClusterVersion(c.Version)
+			if err != nil {
+				return fmt.Errorf("failed to get cluster version for checking cloud provider: %v", err)
+			}
+			// cloud provider must be external or external-aws for >=1.27
+			defaultExternalAwsRange, err := semver.ParseRange(">=1.27.0-rancher0")
+			if err != nil {
+				return fmt.Errorf("failed to parse semver range for checking cloud provider %v", err)
+			}
+			if defaultExternalAwsRange(clusterVersion) {
+				return fmt.Errorf(fmt.Sprintf("Cloud provider %s is invalid for [%s]", aws.AWSCloudProviderName, c.Version))
+			}
 		}
 	}
 	return nil

cluster/plan.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/docker/docker/api/types"
+	"github.com/rancher/rke/cloudprovider/aws"
 	"github.com/rancher/rke/docker"
 	"github.com/rancher/rke/hosts"
 	"github.com/rancher/rke/k8s"
@@ -182,7 +183,7 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 	CommandArgs := map[string]string{
 		"admission-control-config-file": DefaultKubeAPIArgAdmissionControlConfigFileValue,
 		"client-ca-file":                pki.GetCertPath(pki.CACertName),
-		"cloud-provider":                c.CloudProvider.Name,
+		"cloud-provider":                getCloudProviderName(c.CloudProvider.Name),
 		"etcd-cafile":                   etcdCAClientCert,
 		"etcd-certfile":                 etcdClientCert,
 		"etcd-keyfile":                  etcdClientKey,
@@ -347,7 +348,7 @@ func (c *Cluster) BuildKubeAPIProcess(host *hosts.Host, serviceOptions v3.Kubern
 func (c *Cluster) BuildKubeControllerProcess(host *hosts.Host, serviceOptions v3.KubernetesServicesOptions) v3.Process {
 	Command := c.getRKEToolsEntryPoint(host.OS(), "kube-controller-manager")
 	CommandArgs := map[string]string{
-		"cloud-provider":                   c.CloudProvider.Name,
+		"cloud-provider":                   getCloudProviderName(c.CloudProvider.Name),
 		"cluster-cidr":                     c.ClusterCIDR,
 		"kubeconfig":                       pki.GetConfigPath(pki.KubeControllerCertName),
 		"root-ca-file":                     pki.GetCertPath(pki.CACertName),
@@ -466,7 +467,7 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, serviceOptions v3.Kubern
 	Command := c.getRKEToolsEntryPoint(host.OS(), "kubelet")
 	CommandArgs := map[string]string{
 		"client-ca-file": pki.GetCertPath(pki.CACertName),
-		"cloud-provider": c.CloudProvider.Name,
+		"cloud-provider": getCloudProviderName(c.CloudProvider.Name),
 		"cluster-dns":    c.ClusterDNSServer,
 		"cluster-domain": c.ClusterDomain,
 		"fail-swap-on":   strconv.FormatBool(kubelet.FailSwapOn),
@@ -498,6 +499,11 @@ func (c *Cluster) BuildKubeletProcess(host *hosts.Host, serviceOptions v3.Kubern
 		if host.IsWindows() { // compatible with Windows
 			CommandArgs["cloud-config"] = path.Join(host.PrefixPath, cloudConfigFileName)
 		}
+
+		if c.CloudProvider.Name == k8s.ExternalAWSCloudProviderName && c.CloudProvider.UseInstanceMetadataHostname != nil && *c.CloudProvider.UseInstanceMetadataHostname {
+			// rke-tools will inject hostname-override from ec2 instance metadata to match with the spec.nodeName set by cloud provider https://github.com/rancher/rke-tools/blob/3eab4f07aa97a8aeeaaef55b1b7bbc82e2a3374a/entrypoint.sh#L17
+			delete(CommandArgs, "hostname-override")
+		}
 	}
 
 	if c.IsKubeletGenerateServingCertificateEnabled() {
@@ -702,7 +708,8 @@ func (c *Cluster) BuildKubeProxyProcess(host *hosts.Host, serviceOptions v3.Kube
 		} else {
 			CommandArgs["bind-address"] = host.Address
 		}
-		if c.CloudProvider.Name == k8s.AWSCloudProvider && c.CloudProvider.UseInstanceMetadataHostname != nil && *c.CloudProvider.UseInstanceMetadataHostname {
+		if (c.CloudProvider.Name == k8s.ExternalAWSCloudProviderName || c.CloudProvider.Name == aws.AWSCloudProviderName) &&
+			c.CloudProvider.UseInstanceMetadataHostname != nil && *c.CloudProvider.UseInstanceMetadataHostname {
 			// rke-tools will inject hostname-override from ec2 instance metadata to match with the spec.nodeName set by cloud provider https://github.com/rancher/rke-tools/blob/3eab4f07aa97a8aeeaaef55b1b7bbc82e2a3374a/entrypoint.sh#L17
 			delete(CommandArgs, "hostname-override")
 		}
@@ -1319,3 +1326,10 @@ func (c *Cluster) multipleCIDRsConfigured() bool {
 	logrus.Debugf("multipleCIDRsConfigured: Returning false for ClusterCIDR [%s]", c.Services.KubeController.ClusterCIDR)
 	return false
 }
+
+func getCloudProviderName(name string) string {
+	if name == k8s.ExternalAWSCloudProviderName {
+		return "external"
+	}
+	return name
+}

data/data.json

@@ -12066,10 +12066,10 @@
   },
   "v1.25.14-rancher1-1": {
    "etcd": "rancher/mirrored-coreos-etcd:v3.5.6",
-   "alpine": "rancher/rke-tools:v0.1.94",
-   "nginxProxy": "rancher/rke-tools:v0.1.94",
-   "certDownloader": "rancher/rke-tools:v0.1.94",
-   "kubernetesServicesSidecar": "rancher/rke-tools:v0.1.94",
+   "alpine": "rancher/rke-tools:v0.1.95",
+   "nginxProxy": "rancher/rke-tools:v0.1.95",
+   "certDownloader": "rancher/rke-tools:v0.1.95",
+   "kubernetesServicesSidecar": "rancher/rke-tools:v0.1.95",
    "kubedns": "rancher/mirrored-k8s-dns-kube-dns:1.22.8",
    "dnsmasq": "rancher/mirrored-k8s-dns-dnsmasq-nanny:1.22.8",
    "kubednsSidecar": "rancher/mirrored-k8s-dns-sidecar:1.22.8",
@@ -12449,10 +12449,10 @@
   },
   "v1.26.9-rancher1-1": {
    "etcd": "rancher/mirrored-coreos-etcd:v3.5.6",
-   "alpine": "rancher/rke-tools:v0.1.94",
-   "nginxProxy": "rancher/rke-tools:v0.1.94",
-   "certDownloader": "rancher/rke-tools:v0.1.94",
-   "kubernetesServicesSidecar": "rancher/rke-tools:v0.1.94",
+   "alpine": "rancher/rke-tools:v0.1.95",
+   "nginxProxy": "rancher/rke-tools:v0.1.95",
+   "certDownloader": "rancher/rke-tools:v0.1.95",
+   "kubernetesServicesSidecar": "rancher/rke-tools:v0.1.95",
    "kubedns": "rancher/mirrored-k8s-dns-kube-dns:1.22.20",
    "dnsmasq": "rancher/mirrored-k8s-dns-dnsmasq-nanny:1.22.20",
    "kubednsSidecar": "rancher/mirrored-k8s-dns-sidecar:1.22.20",
@@ -12490,10 +12490,10 @@
   },
   "v1.27.6-rancher1-1": {
    "etcd": "rancher/mirrored-coreos-etcd:v3.5.7",
-   "alpine": "rancher/rke-tools:v0.1.94",
-   "nginxProxy": "rancher/rke-tools:v0.1.94",
-   "certDownloader": "rancher/rke-tools:v0.1.94",
-   "kubernetesServicesSidecar": "rancher/rke-tools:v0.1.94",
+   "alpine": "rancher/rke-tools:v0.1.95",
+   "nginxProxy": "rancher/rke-tools:v0.1.95",
+   "certDownloader": "rancher/rke-tools:v0.1.95",
+   "kubernetesServicesSidecar": "rancher/rke-tools:v0.1.95",
    "kubedns": "rancher/mirrored-k8s-dns-kube-dns:1.22.20",
    "dnsmasq": "rancher/mirrored-k8s-dns-dnsmasq-nanny:1.22.20",
    "kubednsSidecar": "rancher/mirrored-k8s-dns-sidecar:1.22.20",

hosts/hosts.go

@@ -191,40 +191,31 @@ func (h *Host) ProcessFilter(processes map[string]v3.Process) map[string]v3.Proc
 	return processes
 }
 
-func DeleteNode(ctx context.Context, toDeleteHost *Host, kubeClient *kubernetes.Clientset, hasAnotherRole bool, cloudProvider string) error {
+func DeleteNode(ctx context.Context, toDeleteHost *Host, kubeClient *kubernetes.Clientset, hasAnotherRole bool, cloudProviderName string) error {
 	if hasAnotherRole {
 		log.Infof(ctx, "[hosts] host [%s] has another role, skipping delete from kubernetes cluster", toDeleteHost.Address)
 		return nil
 	}
 	log.Infof(ctx, "[hosts] Cordoning host [%s]", toDeleteHost.Address)
-	if _, err := k8s.GetNode(kubeClient, toDeleteHost.HostnameOverride); err != nil {
+	if _, err := k8s.GetNode(kubeClient, toDeleteHost.HostnameOverride, cloudProviderName); err != nil {
 		if apierrors.IsNotFound(err) {
 			log.Warnf(ctx, "[hosts] Can't find node by name [%s]", toDeleteHost.Address)
 			return nil
 		}
 		return err
 
 	}
-	if err := k8s.CordonUncordon(kubeClient, toDeleteHost.HostnameOverride, true); err != nil {
+	if err := k8s.CordonUncordon(kubeClient, toDeleteHost.HostnameOverride, cloudProviderName, true); err != nil {
 		return err
 	}
 	log.Infof(ctx, "[hosts] Deleting host [%s] from the cluster", toDeleteHost.Address)
-	if err := k8s.DeleteNode(kubeClient, toDeleteHost.HostnameOverride, cloudProvider); err != nil {
+	if err := k8s.DeleteNode(kubeClient, toDeleteHost.HostnameOverride, cloudProviderName); err != nil {
 		return err
 	}
 	log.Infof(ctx, "[hosts] Successfully deleted host [%s] from the cluster", toDeleteHost.Address)
 	return nil
 }
 
-func RemoveTaintFromHost(ctx context.Context, host *Host, taintKey string, kubeClient *kubernetes.Clientset) error {
-	log.Infof(ctx, "[hosts] removing taint [%s] from host [%s]", taintKey, host.Address)
-	if err := k8s.RemoveTaintFromNodeByKey(kubeClient, host.HostnameOverride, taintKey); err != nil {
-		return err
-	}
-	log.Infof(ctx, "[hosts] Successfully deleted taint [%s] from host [%s]", taintKey, host.Address)
-	return nil
-}
-
 func GetToDeleteHosts(currentHosts, configHosts, inactiveHosts []*Host, includeInactive bool) []*Host {
 	toDeleteHosts := []*Host{}
 	for _, currentHost := range currentHosts {