multus: add support for the Multus daemonset

bindata/network/additional-networks/cr/cr.yaml

@@ -4,4 +4,5 @@ metadata:
   name: {{.AdditionalNetworkName}}
   namespace: default
 spec:
-  config: {{.AdditionalNetworkConfig}}
+  config: |-
+{{.AdditionalNetworkConfig | indent 4}}

bindata/network/multus/000-ns.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: multus
+  labels:
+    name: multus
+    openshift.io/run-level: "0"
+  annotations:
+    openshift.io/node-selector: "" #override default node selector
+    openshift.io/description: "Multus network plugin components"

bindata/network/multus/002-rbac.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources:
+  - customresourcedefinitions
+  - customresourcedefinitions/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - pods
+  - pods/status
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+- apiGroups: [""]
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: multus
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: multus

bindata/network/multus/multus.yaml

@@ -0,0 +1,71 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: multus
+  namespace: multus
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the Multus networking component on each node.
+spec:
+  selector:
+    matchLabels:
+      app: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: multus
+        component: network
+        type: infra
+        openshift.io/component: network
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      tolerations:
+      - operator: Exists
+      serviceAccountName: multus
+      initContainers:
+      - name: cni-plugins-supported
+        image: {{.CNIPluginsSupportedImage}}
+        command: ["/bin/sh"]
+        args: ["-c", "cp -rf /usr/src/plugins/bin/* /host/opt/cni/bin"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cnibin
+      - name: cni-plugins-unsupported
+        image: {{.CNIPluginsUnsupportedImage}}
+        command: ["/bin/sh"]
+        args: ["-c", "cp -rf /usr/src/plugins/bin/* /host/opt/cni/bin"]
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cnibin
+      containers:
+      - name: kube-multus
+        image: {{.MultusImage}}
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        - "--multus-kubeconfig-file-host=/etc/kubernetes/cni/net.d/multus.d/multus.kubeconfig"
+        - "--namespace-isolation=true"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/kubernetes/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /var/lib/cni/bin

manifests/0000_07_cluster-network-operator_03_daemonset.yaml

@@ -29,6 +29,12 @@ spec:
           value: "docker.io/openshift/origin-node:v4.0.0"
         - name: HYPERSHIFT_IMAGE
           value: "docker.io/openshift/origin-hypershift:v4.0.0"
+        - name: MULTUS_IMAGE
+          value: "quay.io/openshift/origin-multus-cni:v4.0.0"
+        - name: CNI_PLUGINS_SUPPORTED_IMAGE
+          value: "quay.io/openshift/origin-container-networking-plugins-supported:v4.0.0"
+        - name: CNI_PLUGINS_UNSUPPORTED_IMAGE
+          value: "quay.io/openshift/origin-container-networking-plugins-unsupported:v4.0.0"
         - name: POD_NAME
           valueFrom:
             fieldRef:

manifests/image-references

@@ -14,3 +14,15 @@ spec:
     from:
       kind: DockerImage
       name: docker.io/openshift/origin-hypershift:v4.0.0
+  - name: multus-cni
+    from:
+      kind: DockerImage
+      name: quay.io/openshift/origin-multus-cni:v4.0.0
+  - name: container-networking-plugins-supported
+    from:
+      kind: DockerImage
+      name: quay.io/openshift/origin-container-networking-plugins-supported:v4.0.0
+  - name: container-networking-plugins-unsupported
+    from:
+      kind: DockerImage
+      name: quay.io/openshift/origin-container-networking-plugins-unsupported:v4.0.0

pkg/apis/networkoperator/v1/networkconfig_types.go

@@ -52,12 +52,15 @@ type NetworkConfigSpec struct {
 	// The "default" network that all pods will receive
 	DefaultNetwork DefaultNetworkDefinition `json:"defaultNetwork"`
 
-	// Additional networks to make available to pods. If they are specified,
-	// pods can request them via annotations.
-	//
-	// Specifying any additionalNetworks will enable Multus across the cluster.
+	// Additional networks to make available to pods when multiple networks
+	// are enabled.
 	AdditionalNetworks []AdditionalNetworkDefinition `json:"additionalNetworks"`
 
+	// DisableMultiNetwork specifies whether or not multiple pod network
+	// support should be disabled. If unset, this property defaults to
+	// 'false' and multiple network support is enabled.
+	DisableMultiNetwork *bool `json:"disableMultiNetwork,omitempty"`
+
 	// DeployKubeProxy specifies whether or not a standalone kube-proxy should
 	// be deployed by the operator. Some network providers include kube-proxy
 	// or similar functionality. If unset, the plugin will attempt to select

pkg/network/additional_networks.go

@@ -9,14 +9,14 @@ import (
 	"path/filepath"
 )
 
-// renderMultusConfig returns the manifests of Multus DaemonSet and NetworkAttachmentDefinition.
-func renderMultusConfig(manifestDir string) ([]*uns.Unstructured, error) {
+// renderAdditionalNetworksCRD returns the manifests of the NetworkAttachmentDefinition.
+func renderAdditionalNetworksCRD(manifestDir string) ([]*uns.Unstructured, error) {
 	objs := []*uns.Unstructured{}
 	// render the manifests on disk
 	data := render.MakeRenderData()
-	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/additional-networks/multus"), &data)
+	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/additional-networks/crd"), &data)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to render manifests")
+		return nil, errors.Wrap(err, "failed to render additional network manifests")
 	}
 	objs = append(objs, manifests...)
 	return objs, nil

pkg/network/additional_networks_test.go

@@ -17,10 +17,10 @@ var NetworkAttachmentConfig = netv1.NetworkConfig{
 	},
 }
 
-func TestRenderMultusConfig(t *testing.T) {
+func TestRenderAdditionalNetworksCRD(t *testing.T) {
 	g := NewGomegaWithT(t)
 
-	objs, err := renderMultusConfig(manifestDir)
+	objs, err := renderAdditionalNetworksCRD(manifestDir)
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(objs).To(HaveLen(1))
 }

pkg/network/multus.go

@@ -0,0 +1,27 @@
+package network
+
+import (
+	"github.com/openshift/cluster-network-operator/pkg/render"
+	"github.com/pkg/errors"
+	uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"os"
+	"path/filepath"
+)
+
+// renderMultusConfig returns the manifests of Multus
+func renderMultusConfig(manifestDir string) ([]*uns.Unstructured, error) {
+	objs := []*uns.Unstructured{}
+
+	// render the manifests on disk
+	data := render.MakeRenderData()
+	data.Data["MultusImage"] = os.Getenv("MULTUS_IMAGE")
+	data.Data["CNIPluginsSupportedImage"] = os.Getenv("CNI_PLUGINS_SUPPORTED_IMAGE")
+	data.Data["CNIPluginsUnsupportedImage"] = os.Getenv("CNI_PLUGINS_UNSUPPORTED_IMAGE")
+
+	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/multus"), &data)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to render multus manifests")
+	}
+	objs = append(objs, manifests...)
+	return objs, nil
+}

pkg/network/multus_test.go

@@ -0,0 +1,75 @@
+package network
+
+import (
+	"testing"
+
+	netv1 "github.com/openshift/cluster-network-operator/pkg/apis/networkoperator/v1"
+	"github.com/openshift/cluster-network-operator/pkg/apply"
+
+	. "github.com/onsi/gomega"
+)
+
+var MultusConfig = netv1.NetworkConfig{
+	Spec: netv1.NetworkConfigSpec{
+		ServiceNetwork: "172.30.0.0/16",
+		ClusterNetworks: []netv1.ClusterNetwork{
+			{
+				CIDR:             "10.128.0.0/15",
+				HostSubnetLength: 9,
+			},
+		},
+		DefaultNetwork: netv1.DefaultNetworkDefinition{
+			Type: netv1.NetworkTypeOpenShiftSDN,
+			OpenShiftSDNConfig: &netv1.OpenShiftSDNConfig{
+				Mode: netv1.SDNModeNetworkPolicy,
+			},
+		},
+	},
+}
+
+// TestRenderMultus has some simple rendering tests
+func TestRenderMultus(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	crd := MultusConfig.DeepCopy()
+	config := &crd.Spec
+	disabled := true
+	config.DisableMultiNetwork = &disabled
+	FillDefaults(config, nil)
+
+	// disable Multus
+	objs, err := RenderMultus(config, manifestDir)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(objs).NotTo(ContainElement(HaveKubernetesID("DaemonSet", "multus", "multus")))
+
+	// enable Multus
+	enabled := false
+	config.DisableMultiNetwork = &enabled
+	objs, err = RenderMultus(config, manifestDir)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "multus", "multus")))
+
+	// It's important that the namespace is first
+	g.Expect(len(objs)).To(Equal(6))
+	g.Expect(objs[0]).To(HaveKubernetesID("CustomResourceDefinition", "", "network-attachment-definitions.k8s.cni.cncf.io"))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("Namespace", "", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ServiceAccount", "multus", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRoleBinding", "", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "multus", "multus")))
+
+	// Make sure every obj is reasonable:
+	// - it is supported
+	// - it reconciles to itself (steady state)
+	for _, obj := range objs {
+		g.Expect(apply.IsObjectSupported(obj)).NotTo(HaveOccurred())
+		cur := obj.DeepCopy()
+		upd := obj.DeepCopy()
+
+		err = apply.MergeObjectForUpdate(cur, upd)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		tweakMetaForCompare(cur)
+		g.Expect(cur).To(Equal(upd))
+	}
+}