multus: add support for the Multus daemonset

Multus will be installed by default but can be turned off by
setting the DeployMultus property of the Cluster Network Operator
object to 'false'.

bindata/network/multus/000-ns.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: multus
+  labels:
+    name: multus
+    openshift.io/run-level: "0"
+  annotations:
+    openshift.io/node-selector: "" #override default node selector
+    openshift.io/description: "Multus network plugin components"

bindata/network/multus/002-rbac.yaml

@@ -0,0 +1,65 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: multus
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources:
+  - customresourcedefinitions
+  - customresourcedefinitions/status
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: ["k8s.cni.cncf.io"]
+  resources: ["*"]
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - namespaces
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups: [""]
+  resources:
+  - pods
+  - pods/status
+  verbs:
+  - get
+  - list
+  - watch
+  - patch
+  - update
+- apiGroups: [""]
+  resources:
+  - events
+  verbs:
+  - create
+  - patch
+  - update
+
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: multus
+  namespace: multus
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: multus
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: multus
+subjects:
+- kind: ServiceAccount
+  name: multus
+  namespace: multus

bindata/network/multus/multus.yaml

@@ -0,0 +1,58 @@
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: multus
+  namespace: multus
+  annotations:
+    kubernetes.io/description: |
+      This daemon set launches the Multus networking component on each node.
+spec:
+  selector:
+    matchLabels:
+      app: multus
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        app: multus
+        component: network
+        type: infra
+        openshift.io/component: network
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+    spec:
+      hostNetwork: true
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+      tolerations:
+      - operator: Exists
+      serviceAccountName: multus
+      containers:
+      - name: kube-multus
+        image: {{.MultusImage}}
+        command: ["/entrypoint.sh"]
+        args:
+        - "--multus-conf-file=auto"
+        resources:
+          requests:
+            cpu: "100m"
+            memory: "50Mi"
+          limits:
+            cpu: "100m"
+            memory: "50Mi"
+        securityContext:
+          privileged: true
+        volumeMounts:
+        - name: cni
+          mountPath: /host/etc/cni/net.d
+        - name: cnibin
+          mountPath: /host/opt/cni/bin
+      volumes:
+        - name: cni
+          hostPath:
+            path: /etc/kubernetes/cni/net.d
+        - name: cnibin
+          hostPath:
+            path: /opt/cni/bin
+

manifests/0000_07_cluster-network-operator_03_daemonset.yaml

@@ -32,6 +32,8 @@ spec:
           value: "docker.io/openshift/origin-node:v4.0.0"
         - name: HYPERSHIFT_IMAGE
           value: "docker.io/openshift/origin-hypershift:v4.0.0"
+        - name: MULTUS_IMAGE
+          value: "docker.io/nfvpe/multus:snapshot"
         - name: POD_NAME
           valueFrom:
             fieldRef:

pkg/apis/networkoperator/v1/networkconfig_types.go

@@ -53,11 +53,13 @@ type NetworkConfigSpec struct {
 	DefaultNetwork DefaultNetworkDefinition `json:"defaultNetwork"`
 
 	// Additional networks to make available to pods. If they are specified,
-	// pods can request them via annotations.
-	//
-	// Specifying any additionalNetworks will enable Multus across the cluster.
+	// pods can request them via annotations when DeployMultus is true.
 	AdditionalNetworks []AdditionalNetworkDefinition `json:"additionalNetworks"`
 
+	// DeployMultus specifies whether or not Multus should be installed to
+	// provide additional network support for pods.
+	DeployMultus *bool `json:"deployMultus,omitempy"`
+
 	// DeployKubeProxy specifies whether or not a standalone kube-proxy should
 	// be deployed by the operator. Some network providers include kube-proxy
 	// or similar functionality. If unset, the plugin will attempt to select

pkg/network/additional_networks.go

@@ -9,14 +9,14 @@ import (
 	"path/filepath"
 )
 
-// renderMultusConfig returns the manifests of Multus DaemonSet and NetworkAttachmentDefinition.
-func renderMultusConfig(manifestDir string) ([]*uns.Unstructured, error) {
+// renderAdditionalNetworksCRD returns the manifests of the NetworkAttachmentDefinition.
+func renderAdditionalNetworksCRD(manifestDir string) ([]*uns.Unstructured, error) {
 	objs := []*uns.Unstructured{}
 	// render the manifests on disk
 	data := render.MakeRenderData()
-	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/additional-networks/multus"), &data)
+	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/additional-networks/crd"), &data)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to render manifests")
+		return nil, errors.Wrap(err, "failed to render additional network manifests")
 	}
 	objs = append(objs, manifests...)
 	return objs, nil

pkg/network/additional_networks_test.go

@@ -17,10 +17,10 @@ var NetworkAttachmentConfig = netv1.NetworkConfig{
 	},
 }
 
-func TestRenderMultusConfig(t *testing.T) {
+func TestRenderAdditionalNetworksCRD(t *testing.T) {
 	g := NewGomegaWithT(t)
 
-	objs, err := renderMultusConfig(manifestDir)
+	objs, err := renderAdditionalNetworksCRD(manifestDir)
 	g.Expect(err).NotTo(HaveOccurred())
 	g.Expect(objs).To(HaveLen(1))
 }

pkg/network/multus.go

@@ -0,0 +1,25 @@
+package network
+
+import (
+	"github.com/openshift/cluster-network-operator/pkg/render"
+	"github.com/pkg/errors"
+	uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"os"
+	"path/filepath"
+)
+
+// renderMultusConfig returns the manifests of Multus
+func renderMultusConfig(manifestDir string) ([]*uns.Unstructured, error) {
+	objs := []*uns.Unstructured{}
+
+	// render the manifests on disk
+	data := render.MakeRenderData()
+	data.Data["MultusImage"] = os.Getenv("MULTUS_IMAGE")
+
+	manifests, err := render.RenderDir(filepath.Join(manifestDir, "network/multus"), &data)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to render multus manifests")
+	}
+	objs = append(objs, manifests...)
+	return objs, nil
+}

pkg/network/multus_test.go

@@ -0,0 +1,90 @@
+package network
+
+import (
+	"testing"
+
+	netv1 "github.com/openshift/cluster-network-operator/pkg/apis/networkoperator/v1"
+	"github.com/openshift/cluster-network-operator/pkg/apply"
+
+	uns "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+
+	. "github.com/onsi/gomega"
+)
+
+var MultusConfig = netv1.NetworkConfig{
+	Spec: netv1.NetworkConfigSpec{
+		ServiceNetwork: "172.30.0.0/16",
+		ClusterNetworks: []netv1.ClusterNetwork{
+			{
+				CIDR:             "10.128.0.0/15",
+				HostSubnetLength: 9,
+			},
+		},
+		DefaultNetwork: netv1.DefaultNetworkDefinition{
+			Type: netv1.NetworkTypeOpenShiftSDN,
+			OpenShiftSDNConfig: &netv1.OpenShiftSDNConfig{
+				Mode: netv1.SDNModeNetworkPolicy,
+			},
+		},
+	},
+}
+
+// TestRenderMultus has some simple rendering tests
+func TestRenderMultus(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	crd := MultusConfig.DeepCopy()
+	config := &crd.Spec
+	off := false
+	config.DeployMultus = &off
+	FillDefaults(config, nil)
+
+	// disable Multus
+	objs, err := RenderMultus(config, manifestDir)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(objs).NotTo(ContainElement(HaveKubernetesID("DaemonSet", "multus", "multus")))
+
+	// enable Multus
+	on := true
+	config.DeployMultus = &on
+	objs, err = RenderMultus(config, manifestDir)
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "multus", "multus")))
+
+	// It's important that the namespace is first
+	g.Expect(len(objs)).To(Equal(5))
+	g.Expect(objs[0]).To(HaveKubernetesID("Namespace", "", "multus"))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRole", "", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ServiceAccount", "multus", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("ClusterRoleBinding", "", "multus")))
+	g.Expect(objs).To(ContainElement(HaveKubernetesID("DaemonSet", "multus", "multus")))
+
+	// make sure all deployments are in the master
+	for _, obj := range objs {
+		if obj.GetKind() != "Deployment" {
+			continue
+		}
+
+		sel, found, err := uns.NestedStringMap(obj.Object, "spec", "template", "spec", "nodeSelector")
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(found).To(BeTrue())
+
+		_, ok := sel["node-role.kubernetes.io/master"]
+		g.Expect(ok).To(BeTrue())
+	}
+
+	// Make sure every obj is reasonable:
+	// - it is supported
+	// - it reconciles to itself (steady state)
+	for _, obj := range objs {
+		g.Expect(apply.IsObjectSupported(obj)).NotTo(HaveOccurred())
+		cur := obj.DeepCopy()
+		upd := obj.DeepCopy()
+
+		err = apply.MergeObjectForUpdate(cur, upd)
+		g.Expect(err).NotTo(HaveOccurred())
+
+		tweakMetaForCompare(cur)
+		g.Expect(cur).To(Equal(upd))
+	}
+}