Skip to content

Commit

Permalink
make azure storage account private (#255)
Browse files Browse the repository at this point in the history
* splits azure storage into smaller files and add update function

* small update

* make azure storage account private

* add-private-dns-zone

* add private mode flag

* bump deps

* edit changelog
  • Loading branch information
QuentinBisson authored Jan 7, 2025
1 parent 46f8c6f commit 399711c
Show file tree
Hide file tree
Showing 9 changed files with 367 additions and 100 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

### Changed

- Secure Azure Storage Account by making them private and accessible through an Azure Private Endpoint. This also requires the creation of a private DNS zone and A record.
- Update Kyverno PolicyException to v2beta1.

### Removed
Expand Down
6 changes: 3 additions & 3 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@ module github.com/giantswarm/object-storage-operator

go 1.23.0

toolchain go1.23.4

require (
github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0
github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0
github.com/aquilax/truncate v1.0.1
github.com/aws/aws-sdk-go-v2 v1.32.7
Expand All @@ -23,6 +23,7 @@ require (
github.com/onsi/ginkgo/v2 v2.22.2
github.com/onsi/gomega v1.36.2
github.com/pkg/errors v0.9.1
gopkg.in/yaml.v3 v3.0.1
k8s.io/api v0.32.0
k8s.io/apimachinery v0.32.0
k8s.io/client-go v0.32.0
Expand Down Expand Up @@ -95,7 +96,6 @@ require (
google.golang.org/protobuf v1.36.2 // indirect
gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
k8s.io/apiextensions-apiserver v0.32.0 // indirect
k8s.io/klog/v2 v2.130.1 // indirect
k8s.io/kube-openapi v0.0.0-20241212222426-2c72e554b1e7 // indirect
Expand Down
8 changes: 6 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,12 @@ github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0 h1:+m0M/LFxN43KvUL
github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.0/go.mod h1:PwOyop78lveYMRs6oCxjiVyBdyCgIYH6XHIVZO9/SFQ=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY=
github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.0.0 h1:Kb8eVvjdP6kZqYnER5w/PiGCFp91yVgaxve3d7kCEpY=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.0.0/go.mod h1:lYq15QkJyEsNegz5EhI/0SXQ6spvGfgwBH/Qyzkoc/s=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0 h1:2qsIIvxVT+uE6yrNldntJKlLRgxGbZ85kgtz5SNBhMw=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/internal/v3 v3.1.0/go.mod h1:AW8VEadnhw9xox+VaVd9sP7NjzOAnaZBLRH6Tq3cJ38=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0 h1:HYGD75g0bQ3VO/Omedm54v4LrD3B1cGImuRF3AJ5wLo=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6 v6.2.0/go.mod h1:ulHyBFJOI0ONiRL4vcJTmS7rx18jQQlEPmAgo80cRdM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0 h1:yzrctSl9GMIQ5lHu7jc8olOsGjWDCsBpJhWqfGa/YIM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns v1.3.0/go.mod h1:GE4m0rnnfwLGX0Y9A9A25Zx5N/90jneT5ABevqzhuFQ=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0 h1:Dd+RhdJn0OTtVGaeDLZpcumkIVCtA/3/Fo42+eoYvVM=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resources/armresources v1.2.0/go.mod h1:5kakwfW5CjC9KK+Q4wjXAg+ShuIm2mBMua0ZFj2C8PE=
github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c=
Expand Down
77 changes: 32 additions & 45 deletions helm/object-storage-operator/templates/rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,21 +5,7 @@ metadata:
{{- include "labels.common" . | nindent 4 }}
name: {{ include "resource.default.name" . }}
rules:
- apiGroups:
- infrastructure.cluster.x-k8s.io
{{ if eq .Values.managementCluster.provider.kind "capa" -}}
resources:
- awsclusters
- awsclusterroleidentities
{{ else if eq .Values.managementCluster.provider.kind "capz" -}}
resources:
- azureclusters
- azureclusteridentities
{{- end }}
verbs:
- get
- list
- watch
# RBAC needed for all providers
- apiGroups:
- objectstorage.giantswarm.io
resources:
Expand All @@ -45,48 +31,40 @@ rules:
- events
verbs:
- create
{{ if eq .Values.managementCluster.provider.kind "capz" -}}

{{ if eq .Values.managementCluster.provider.kind "capa" -}}
- apiGroups:
- infrastructure.cluster.x-k8s.io
resources:
- azureclusteridentities
- azureclusteridentities/status
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- aadpodidentity.k8s.io
- infrastructure.cluster.x-k8s.io
resources:
- azureidentities
- azureidentities/status
- awsclusters
- awsclusterroleidentities
verbs:
- get
- list
- watch
- get
- list
- watch
{{- end}}

{{ if eq .Values.managementCluster.provider.kind "capz" -}}
- apiGroups:
- aadpodidentity.k8s.io
- infrastructure.cluster.x-k8s.io
resources:
- azureidentities
- azureclusters
verbs:
- create
- get
- list
- watch
- apiGroups:
- aadpodidentity.k8s.io
- infrastructure.cluster.x-k8s.io
resources:
- azureidentitybindings
- azureidentitybindings/status
- azureclusteridentities
- azureclusteridentities/status
verbs:
- get
- list
- patch
- update
- watch
- apiGroups:
- aadpodidentity.k8s.io
resources:
- azureidentitybindings
verbs:
- create
# Needed to store Azure storage account credentials
- apiGroups:
- ""
resources:
Expand All @@ -99,6 +77,15 @@ rules:
- get
- list
- watch
# Needed for Azure to detect private mode
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
{{- end }}
---
apiVersion: rbac.authorization.k8s.io/v1
Expand Down
8 changes: 8 additions & 0 deletions internal/pkg/service/objectstorage/cloud/azure/cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -201,3 +201,11 @@ func (c AzureCluster) GetCredentials() cluster.Credentials {
func (c AzureCluster) GetResourceGroup() string {
return c.Credentials.ResourceGroup
}

func (c AzureCluster) GetSubscriptionID() string {
return c.Credentials.SubscriptionID
}

func (c AzureCluster) GetVNetName() string {
return c.GetName() + "-vnet"
}
158 changes: 158 additions & 0 deletions internal/pkg/service/objectstorage/cloud/azure/privateendpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
package azure

import (
"context"
"fmt"
"time"

"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"

"github.com/giantswarm/object-storage-operator/api/v1alpha1"
)

const (
// This is how the private zone must be named in Azure for the private endpoint to work.
privateZoneID = "privatelink.blob.core.windows.net"
vnetID = "/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Network/virtualNetworks/%s"
subnetID = vnetID + "/subnets/%s"
)

func (s AzureObjectStorageAdapter) upsertPrivateEndpointARecords(ctx context.Context, bucket *v1alpha1.Bucket, privateEndpoint *armnetwork.PrivateEndpoint, storageAccountName string) (*armprivatedns.RecordSet, error) {
s.logger.Info("Creating A record for private endpoint", "private-endpoint", *privateEndpoint.Name)

ips := make([]string, 0)

for _, dnsConfigs := range privateEndpoint.Properties.CustomDNSConfigs {
if dnsConfigs.IPAddresses == nil {
continue
}
for _, ip := range dnsConfigs.IPAddresses {
ips = append(ips, *ip)
}
}

aRecords := make([]*armprivatedns.ARecord, len(ips))
for i, ip := range ips {
aRecords[i] = &armprivatedns.ARecord{IPv4Address: &ip}
}

resp, err := s.recordSetsClient.CreateOrUpdate(
ctx,
s.cluster.GetResourceGroup(),
privateZoneID,
armprivatedns.RecordTypeA,
storageAccountName,
armprivatedns.RecordSet{
Properties: &armprivatedns.RecordSetProperties{
ARecords: aRecords,
TTL: to.Ptr(int64(time.Hour.Seconds())),
Metadata: s.getBucketTags(bucket),
},
},
nil,
)
if err != nil {
return nil, err
}
return &resp.RecordSet, nil
}

func (s AzureObjectStorageAdapter) upsertPrivateEndpoint(ctx context.Context, bucket *v1alpha1.Bucket, storageAccountName string) (*armnetwork.PrivateEndpoint, error) {
// Create or Update Private endpoint
pollersResp, err := s.privateEndpointsClient.BeginCreateOrUpdate(
ctx,
s.cluster.GetResourceGroup(),
bucket.Spec.Name,
armnetwork.PrivateEndpoint{
Location: to.Ptr(s.cluster.GetRegion()),
Properties: &armnetwork.PrivateEndpointProperties{
CustomNetworkInterfaceName: to.Ptr(fmt.Sprintf("%s-nodes-nic", bucket.Spec.Name)),
PrivateLinkServiceConnections: []*armnetwork.PrivateLinkServiceConnection{
{
Name: to.Ptr(bucket.Spec.Name),
Properties: &armnetwork.PrivateLinkServiceConnectionProperties{
PrivateLinkServiceID: to.Ptr(fmt.Sprintf("/subscriptions/%s/resourceGroups/%s/providers/Microsoft.Storage/storageAccounts/%s", s.cluster.GetSubscriptionID(), s.cluster.GetResourceGroup(), storageAccountName)),
GroupIDs: []*string{to.Ptr("blob")},
},
},
},
Subnet: &armnetwork.Subnet{
ID: to.Ptr(s.subnetID()),
},
},
Tags: s.getBucketTags(bucket),
},
nil,
)

if err != nil {
return nil, err
}
resp, err := pollersResp.PollUntilDone(ctx, nil)
if err != nil {
return nil, err
}

return &resp.PrivateEndpoint, nil
}

func (s AzureObjectStorageAdapter) upsertPrivateZone(ctx context.Context, bucket *v1alpha1.Bucket) (*armprivatedns.PrivateZone, error) {
pollersResp, err := s.privateZonesClient.BeginCreateOrUpdate(
ctx,
s.cluster.GetResourceGroup(),
privateZoneID,
armprivatedns.PrivateZone{
// Private Zone DNS is a global resource
Location: to.Ptr("Global"),
Tags: s.getBucketTags(bucket),
},
nil,
)
if err != nil {
return nil, err
}
resp, err := pollersResp.PollUntilDone(ctx, nil)
if err != nil {
return nil, err
}
return &resp.PrivateZone, nil
}

func (s AzureObjectStorageAdapter) upsertVirtualNetworkLink(ctx context.Context, bucket *v1alpha1.Bucket) (*armprivatedns.VirtualNetworkLink, error) {
pollersResp, err := s.virtualNetworkLinksClient.BeginCreateOrUpdate(
ctx,
s.cluster.GetResourceGroup(),
privateZoneID,
"giantswarm-observability",
armprivatedns.VirtualNetworkLink{
// Private Zone DNS is a global resource
Location: to.Ptr("Global"),
Properties: &armprivatedns.VirtualNetworkLinkProperties{
RegistrationEnabled: to.Ptr(false),
VirtualNetwork: &armprivatedns.SubResource{
ID: to.Ptr(s.vnetID()),
},
},
Tags: s.getBucketTags(bucket),
},
nil,
)
if err != nil {
return nil, err
}
resp, err := pollersResp.PollUntilDone(ctx, nil)
if err != nil {
return nil, err
}
return &resp.VirtualNetworkLink, nil
}

func (s AzureObjectStorageAdapter) vnetID() string {
return fmt.Sprintf(vnetID, s.cluster.GetSubscriptionID(), s.cluster.GetResourceGroup(), s.cluster.GetVNetName())
}

func (s AzureObjectStorageAdapter) subnetID() string {
return fmt.Sprintf(subnetID, s.cluster.GetSubscriptionID(), s.cluster.GetResourceGroup(), s.cluster.GetVNetName(), "node-subnet")
}
27 changes: 26 additions & 1 deletion internal/pkg/service/objectstorage/cloud/azure/service.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (

"github.com/Azure/azure-sdk-for-go/sdk/azcore"
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/network/armnetwork/v6"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/privatedns/armprivatedns"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage"
"github.com/go-logr/logr"
"github.com/pkg/errors"
Expand Down Expand Up @@ -57,9 +59,32 @@ func (s AzureObjectStorageService) NewObjectStorageService(ctx context.Context,
return nil, errors.WithStack(err)
}

var networkClientFactory *armnetwork.ClientFactory
networkClientFactory, err = armnetwork.NewClientFactory(azureCredentials.SubscriptionID, cred, nil)
if err != nil {
return nil, errors.WithStack(err)
}

var privateZonesClientFactory *armprivatedns.ClientFactory
privateZonesClientFactory, err = armprivatedns.NewClientFactory(azureCredentials.SubscriptionID, cred, nil)
if err != nil {
return nil, errors.WithStack(err)
}

azurecluster, ok := cluster.(AzureCluster)
if !ok {
return nil, errors.New("Impossible to cast cluster into Azure cluster")
}
return NewAzureStorageService(storageClientFactory.NewAccountsClient(), storageClientFactory.NewBlobContainersClient(), storageClientFactory.NewManagementPoliciesClient(), logger, azurecluster, client), nil
return NewAzureStorageService(
storageClientFactory.NewAccountsClient(),
storageClientFactory.NewBlobContainersClient(),
storageClientFactory.NewManagementPoliciesClient(),
networkClientFactory.NewPrivateEndpointsClient(),
privateZonesClientFactory.NewPrivateZonesClient(),
privateZonesClientFactory.NewRecordSetsClient(),
privateZonesClientFactory.NewVirtualNetworkLinksClient(),
logger,
azurecluster,
client,
), nil
}
Loading

0 comments on commit 399711c

Please sign in to comment.