Creates VPEs as needed for Disconnected install

Signed-off-by: Hiro Miyamoto <[email protected]>

pkg/asset/cluster/metadata.go

@@ -3,6 +3,7 @@ package cluster
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 
 	"github.com/pkg/errors"
 
@@ -97,7 +98,10 @@ func (m *Metadata) Generate(_ context.Context, parents asset.Parents) (err error
 	case vspheretypes.Name:
 		metadata.ClusterPlatformMetadata.VSphere = vsphere.Metadata(installConfig.Config)
 	case powervstypes.Name:
-		metadata.ClusterPlatformMetadata.PowerVS = powervs.Metadata(installConfig.Config, installConfig.PowerVS)
+		metadata.ClusterPlatformMetadata.PowerVS, err = powervs.Metadata(installConfig.Config, installConfig.PowerVS)
+		if err != nil {
+			return fmt.Errorf("failed to initialize PowerVS: %w", err)
+		}
 	case externaltypes.Name, nonetypes.Name:
 	case nutanixtypes.Name:
 		metadata.ClusterPlatformMetadata.Nutanix = nutanix.Metadata(installConfig.Config)

pkg/asset/cluster/powervs/powervs.go

@@ -10,10 +10,24 @@ import (
 )
 
 // Metadata converts an install configuration to PowerVS metadata.
-func Metadata(config *types.InstallConfig, meta *icpowervs.Metadata) *powervs.Metadata {
+func Metadata(config *types.InstallConfig, meta *icpowervs.Metadata) (*powervs.Metadata, error) {
 	cisCRN, _ := meta.CISInstanceCRN(context.TODO())
 	dnsCRN, _ := meta.DNSInstanceCRN(context.TODO())
 
+	overrides := config.Platform.PowerVS.ServiceEndpoints
+	if config.Publish == types.InternalPublishingStrategy &&
+		(len(config.ImageDigestSources) > 0 || len(config.DeprecatedImageContentSources) > 0) {
+		cosRegion, err := powervs.COSRegionForPowerVSRegion(config.PowerVS.Region)
+		if err != nil {
+			return nil, err
+		}
+		vpcRegion, err := powervs.VPCRegionForPowerVSRegion(config.PowerVS.Region)
+		if err != nil {
+			return nil, err
+		}
+		overrides = meta.SetDefaultPrivateServiceEndpoints(context.TODO(), overrides, cosRegion, vpcRegion)
+	}
+
 	return &powervs.Metadata{
 		BaseDomain:           config.BaseDomain,
 		PowerVSResourceGroup: config.Platform.PowerVS.PowerVSResourceGroup,
@@ -23,7 +37,7 @@ func Metadata(config *types.InstallConfig, meta *icpowervs.Metadata) *powervs.Me
 		VPCRegion:            config.Platform.PowerVS.VPCRegion,
 		Zone:                 config.Platform.PowerVS.Zone,
 		ServiceInstanceGUID:  config.Platform.PowerVS.ServiceInstanceGUID,
-		ServiceEndpoints:     config.Platform.PowerVS.ServiceEndpoints,
+		ServiceEndpoints:     overrides,
 		TransitGatewayName:   config.Platform.PowerVS.TGName,
-	}
+	}, nil
 }

pkg/asset/installconfig/powervs/client.go

@@ -85,6 +85,9 @@ type API interface {
 
 	// Load Balancer
 	AddIPToLoadBalancerPool(ctx context.Context, lbID string, poolName string, port int64, ip string) error
+
+	// Virtual Private Endpoint Gateway
+	CreateVirtualPrivateEndpointGateway(ctx context.Context, name string, vpcID string, subnetID string, rgID string, targetCRN string) (*vpcv1.EndpointGateway, error)
 }
 
 // Client makes calls to the PowerVS API.
@@ -1459,3 +1462,73 @@ func (c *Client) AddIPToLoadBalancerPool(ctx context.Context, lbID string, poolN
 			return true, nil
 		})
 }
+
+// CreateVirtualPrivateEndpointGateway creates a VPE gateway with given target resource type and CRN.
+func (c *Client) CreateVirtualPrivateEndpointGateway(ctx context.Context, name string, vpcID string, subnetID string, rgID string, targetCRN string) (*vpcv1.EndpointGateway, error) {
+	var (
+		resp   *core.DetailedResponse
+		err    error
+		ok     bool
+		egs    *vpcv1.EndpointGatewayCollection
+		egRef  *vpcv1.EndpointGatewayTarget
+		idIntf *vpcv1.VPCIdentityByID
+		target *vpcv1.EndpointGatewayTargetPrototypeEndpointGatewayTargetResourceTypeProviderCloudServicePrototype
+		rgIntf *vpcv1.ResourceGroupIdentityByID
+		ipIntf *vpcv1.EndpointGatewayReservedIPReservedIPIdentityByID
+	)
+
+	listOpts := c.vpcAPI.NewListEndpointGatewaysOptions()
+	listOpts.SetVPCID(vpcID)
+	egs, _, err = c.vpcAPI.ListEndpointGateways(listOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, eg := range egs.EndpointGateways {
+		egRef, ok = eg.Target.(*vpcv1.EndpointGatewayTarget)
+		if !ok {
+			return nil, fmt.Errorf("invalid target inside returned EndpointGateway: %v", eg.Target)
+		}
+		if *egRef.CRN == targetCRN {
+			return &eg, nil
+		}
+	}
+
+	target, err = c.vpcAPI.NewEndpointGatewayTargetPrototypeEndpointGatewayTargetResourceTypeProviderCloudServicePrototype(targetCRN, vpcv1.EndpointGatewayTargetPrototypeResourceTypeProviderCloudServiceConst)
+	if err != nil {
+		return nil, err
+	}
+	idIntf, err = c.vpcAPI.NewVPCIdentityByID(vpcID)
+	if err != nil {
+		return nil, err
+	}
+	createOpts := c.vpcAPI.NewCreateEndpointGatewayOptions(target, idIntf)
+	createOpts.SetName(name)
+	createOpts.SetAllowDnsResolutionBinding(true)
+	rgIntf, err = c.vpcAPI.NewResourceGroupIdentityByID(rgID)
+	if err != nil {
+		return nil, err
+	}
+	createOpts.SetResourceGroup(rgIntf)
+	ipName := fmt.Sprintf("%s-ip", name)
+	createIPOpts := c.vpcAPI.NewCreateSubnetReservedIPOptions(subnetID)
+	createIPOpts.SetName(ipName)
+	createIPOpts.SetSubnetID(subnetID)
+	reservedIP, _, err := c.vpcAPI.CreateSubnetReservedIPWithContext(ctx, createIPOpts)
+	if err != nil {
+		return nil, err
+	}
+	ipIntf, err = c.vpcAPI.NewEndpointGatewayReservedIPReservedIPIdentityByID(*reservedIP.ID)
+	if err != nil {
+		return nil, err
+	}
+	ips := []vpcv1.EndpointGatewayReservedIPIntf{ipIntf}
+	createOpts.SetIps(ips)
+
+	eg, resp, err := c.vpcAPI.CreateEndpointGatewayWithContext(ctx, createOpts)
+	if err != nil {
+		logrus.Debugf("CreateEndpointGatewayWithContext returned %v", resp)
+		return nil, err
+	}
+	return eg, nil
+}

pkg/asset/installconfig/powervs/metadata.go

@@ -9,11 +9,21 @@ import (
 
 	"github.com/IBM-Cloud/bluemix-go/crn"
 	"github.com/IBM/vpc-go-sdk/vpcv1"
+	"github.com/sirupsen/logrus"
 	"k8s.io/apimachinery/pkg/util/wait"
 
+	configv1 "github.com/openshift/api/config/v1"
 	"github.com/openshift/installer/pkg/types"
 )
 
+const (
+	cosCrnTempl = "crn:v1:bluemix:public:cloud-object-storage:global:::endpoint:s3.direct.%s.cloud-object-storage.appdomain.cloud"
+	dnsCrn      = "crn:v1:bluemix:public:dns-svcs:global::::"
+	iamCrn      = "crn:v1:bluemix:public:iam-svcs:global:::endpoint:private.iam.cloud.ibm.com"
+	rcCrn       = "crn:v1:bluemix:public:resource-controller:global:::endpoint:private.resource-controller.cloud.ibm.com"
+	vpcCrnTempl = "crn:v1:bluemix:public:is:%s:::endpoint:%s.private.iaas.cloud.ibm.com"
+)
+
 //go:generate mockgen -source=./metadata.go -destination=./mock/powervsmetadata_generated.go -package=mock
 
 // MetadataAPI represents functions that eventually call out to the API
@@ -491,3 +501,117 @@ func leftInContext(ctx context.Context) time.Duration {
 
 	return time.Until(deadline)
 }
+
+// SetDefaultPrivateServiceEndpoints sets service endpoint overrides as needed for Disconnected install.
+func (m *Metadata) SetDefaultPrivateServiceEndpoints(ctx context.Context, overrides []configv1.PowerVSServiceEndpoint, cosRegion string, vpcRegion string) []configv1.PowerVSServiceEndpoint {
+	overrides = addOverride(overrides, "COS", fmt.Sprintf("https://s3.direct.%s.cloud-object-storage.appdomain.cloud", cosRegion))
+	overrides = addOverride(overrides, "DNSServices", "https://api.private.dns-svcs.cloud.ibm.com")
+	overrides = addOverride(overrides, "IAM", "https://private.iam.cloud.ibm.com")
+	overrides = addOverride(overrides, "Power", fmt.Sprintf("https://private.%s.power-iaas.cloud.ibm.com", vpcRegion))
+	overrides = addOverride(overrides, "ResourceController", "https://private.resource-controller.cloud.ibm.com")
+	overrides = addOverride(overrides, "VPC", fmt.Sprintf("https://%s.private.iaas.cloud.ibm.com", vpcRegion))
+	return overrides
+}
+
+func addOverride(overrides []configv1.PowerVSServiceEndpoint, name string, url string) []configv1.PowerVSServiceEndpoint {
+	found := false
+	for _, service := range overrides {
+		if service.Name == name {
+			found = true
+			break
+		}
+	}
+	if !found {
+		return append(overrides, configv1.PowerVSServiceEndpoint{Name: name, URL: url})
+	}
+	return overrides
+}
+
+// CreateVirtualPrivateEndpointGateways checks and creates necessary VPEs in given target VPC for Disconnected install.
+func (m *Metadata) CreateVirtualPrivateEndpointGateways(ctx context.Context, infraID string, region string, vpcID string, subnetID string, groupID string, endpointOverrides []configv1.PowerVSServiceEndpoint) error {
+	m.mutex.Lock()
+	defer m.mutex.Unlock()
+
+	client, err := m.client()
+	if err != nil {
+		return err
+	}
+
+	endpoint, err := m.fetchEndpointOverride(endpointOverrides, "IAM")
+	if err != nil {
+		return fmt.Errorf("failed to fetch endpoint override found for IAM: %w", err)
+	}
+	if endpoint == "" {
+		name := fmt.Sprintf("%s-vpe-iam", infraID)
+		logrus.Debugf("InfraReady: Ensuring VPE gateway for IAM %v", iamCrn)
+		_, err = client.CreateVirtualPrivateEndpointGateway(ctx, name, vpcID, subnetID, groupID, iamCrn)
+		if err != nil {
+			return fmt.Errorf("failed to create VPE: %w", err)
+		}
+	}
+
+	endpoint, err = m.fetchEndpointOverride(endpointOverrides, "COS")
+	if err != nil {
+		return fmt.Errorf("failed to fetch endpoint override found for COS: %w", err)
+	}
+	if endpoint == "" {
+		name := fmt.Sprintf("%s-vpe-cos", infraID)
+		cosCrn := fmt.Sprintf(cosCrnTempl, region)
+		logrus.Debugf("InfraReady: Ensuring VPE gateway for COS %v", cosCrn)
+		_, err = client.CreateVirtualPrivateEndpointGateway(ctx, name, vpcID, subnetID, groupID, cosCrn)
+		if err != nil {
+			return fmt.Errorf("failed to create VPE: %w", err)
+		}
+	}
+
+	endpoint, err = m.fetchEndpointOverride(endpointOverrides, "DNSServices")
+	if err != nil {
+		return fmt.Errorf("failed to fetch endpoint override found for DNS: %w", err)
+	}
+	if endpoint == "" {
+		name := fmt.Sprintf("%s-vpe-dns", infraID)
+		logrus.Debugf("InfraReady: Ensuring VPE gateway for DNS services %v", dnsCrn)
+		_, err = client.CreateVirtualPrivateEndpointGateway(ctx, name, vpcID, subnetID, groupID, dnsCrn)
+		if err != nil {
+			return fmt.Errorf("failed to create VPE: %w", err)
+		}
+	}
+
+	endpoint, err = m.fetchEndpointOverride(endpointOverrides, string(configv1.IBMCloudServiceResourceController))
+	if err != nil {
+		return fmt.Errorf("failed to fetch endpoint override found for RC: %w", err)
+	}
+	if endpoint == "" {
+		name := fmt.Sprintf("%s-vpe-rc", infraID)
+		logrus.Debugf("InfraReady: Ensuring VPE gateway for RC %v", rcCrn)
+		_, err = client.CreateVirtualPrivateEndpointGateway(ctx, name, vpcID, subnetID, groupID, rcCrn)
+		if err != nil {
+			return fmt.Errorf("failed to create VPE: %w", err)
+		}
+	}
+
+	endpoint, err = m.fetchEndpointOverride(endpointOverrides, string(configv1.IBMCloudServiceVPC))
+	if err != nil {
+		return fmt.Errorf("failed to fetch endpoint override found for VPC: %w", err)
+	}
+	if endpoint == "" {
+		name := fmt.Sprintf("%s-vpe-vpc", infraID)
+		vpcCrn := fmt.Sprintf(vpcCrnTempl, region, region)
+		logrus.Debugf("InfraReady: Ensuring VPE gateway for VPC %v", vpcCrn)
+		_, err = client.CreateVirtualPrivateEndpointGateway(ctx, name, vpcID, subnetID, groupID, vpcCrn)
+		if err != nil {
+			return fmt.Errorf("failed to create VPE: %w", err)
+		}
+	}
+
+	return nil
+}
+
+func (m *Metadata) fetchEndpointOverride(overrides []configv1.PowerVSServiceEndpoint, svcID string) (string, error) {
+	for _, endpoint := range overrides {
+		if endpoint.Name == svcID {
+			return endpoint.URL, nil
+		}
+	}
+	return "", nil
+}

pkg/asset/installconfig/powervs/session.go

@@ -15,7 +15,6 @@ import (
 	"github.com/IBM/go-sdk-core/v5/core"
 	"github.com/form3tech-oss/jwt-go"
 	"github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/openshift/installer/pkg/types/powervs"
 )
@@ -427,14 +426,22 @@ func getEnv(envs []string) string {
 	return ""
 }
 
-// FilterServiceEndpoints drops service endpoint overrides that are not supported by PowerVS CAPI provider.
-func (c *BxClient) FilterServiceEndpoints(cfg *powervs.Metadata) []string {
-	capiSupported := sets.New("cos", "powervs", "rc", "rm", "vpc") // see serviceIDs array definition in https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/pkg/endpoints/endpoints.go
+// MapServiceEndpointsForCAPI drops service endpoint overrides that are not supported by PowerVS CAPI provider, while also translating service names.
+func (c *BxClient) MapServiceEndpointsForCAPI(cfg *powervs.Metadata) []string {
+	// Keys are what installer recognizes from install-config.yaml, and values are what PowerVS CAPI accepts
+	// Should contain only mapping for serviceIDs from https://github.com/kubernetes-sigs/cluster-api-provider-ibmcloud/blob/main/pkg/endpoints/endpoints.go
+	capiSupported := map[string]string{
+		"COS":                "cos",
+		"Power":              "powervs",
+		"ResourceController": "", // FIXME CAPI recognizes "rc," but crashes if passed in...
+		"ResourceManager":    "rm",
+		"VPC":                "vpc",
+	}
 	overrides := make([]string, 0, len(cfg.ServiceEndpoints))
 	// CAPI expects name=url pairs of service endpoints
 	for _, endpoint := range cfg.ServiceEndpoints {
-		if capiSupported.Has(endpoint.Name) {
-			overrides = append(overrides, fmt.Sprintf("%s=%s", endpoint.Name, endpoint.URL))
+		if capiName, ok := capiSupported[endpoint.Name]; ok && capiName != "" {
+			overrides = append(overrides, fmt.Sprintf("%s=%s", capiSupported[endpoint.Name], endpoint.URL))
 		} else {
 			logrus.Infof("Unsupported service endpoint skipped: %s", endpoint.Name)
 		}

pkg/asset/manifests/cloudproviderconfig.go

@@ -300,6 +300,11 @@ func (cpc *CloudProviderConfig) Generate(ctx context.Context, dependencies asset
 			serviceGUID = installConfig.Config.PowerVS.ServiceInstanceGUID
 		}
 
+		cosRegion, err := powervstypes.COSRegionForPowerVSRegion(installConfig.Config.PowerVS.Region)
+		if err != nil {
+			return err
+		}
+
 		powervsConfig, err := powervsmanifests.CloudProviderConfig(
 			clusterID.InfraID,
 			accountID,
@@ -311,7 +316,7 @@ func (cpc *CloudProviderConfig) Generate(ctx context.Context, dependencies asset
 			serviceName,
 			installConfig.Config.PowerVS.Region,
 			installConfig.Config.PowerVS.Zone,
-			installConfig.Config.PowerVS.ServiceEndpoints,
+			installConfig.PowerVS.SetDefaultPrivateServiceEndpoints(ctx, installConfig.Config.PowerVS.ServiceEndpoints, cosRegion, vpcRegion),
 		)
 		if err != nil {
 			return errors.Wrap(err, "could not create cloud provider config")

pkg/asset/manifests/infrastructure.go

@@ -311,6 +311,19 @@ func (i *Infrastructure) Generate(ctx context.Context, dependencies asset.Parent
 				URL:  service.URL,
 			})
 		}
+		if installConfig.Config.Publish == types.InternalPublishingStrategy &&
+			(len(installConfig.Config.ImageDigestSources) > 0 || len(installConfig.Config.DeprecatedImageContentSources) > 0) {
+			piRegion := installConfig.Config.PowerVS.Region
+			vpcRegion, err := powervs.VPCRegionForPowerVSRegion(piRegion)
+			if err != nil {
+				return fmt.Errorf("failed to determine VPC region: %w", err)
+			}
+			cosRegion, err := powervs.COSRegionForPowerVSRegion(piRegion)
+			if err != nil {
+				return fmt.Errorf("failed to determine COS region: %w", err)
+			}
+			config.Spec.PlatformSpec.PowerVS.ServiceEndpoints = installConfig.PowerVS.SetDefaultPrivateServiceEndpoints(ctx, config.Spec.PlatformSpec.PowerVS.ServiceEndpoints, cosRegion, vpcRegion)
+		}
 		config.Status.PlatformStatus.PowerVS = &configv1.PowerVSPlatformStatus{
 			Region:           installConfig.Config.Platform.PowerVS.Region,
 			Zone:             installConfig.Config.Platform.PowerVS.Zone,