feat: Blacklist DOCA Drive modules on host (#1088)

We should not allow to load inbox or OFED-on-host drivers if DOCA Driver container deployment is enabled. Signed-off-by: Ivan Kolodiazhnyi <[email protected]> (cherry picked from commit ea72766)
Mellanox · Oct 1, 2024 · 2800c25 · 2800c25
2 parents b96970b + efd8897
commit 2800c25
Show file tree

Hide file tree

Showing 3 changed files with 53 additions and 10 deletions.
diff --git a/manifests/state-ofed-driver/0050_ofed-modules-blacklist-configmap.yaml b/manifests/state-ofed-driver/0050_ofed-modules-blacklist-configmap.yaml
@@ -0,0 +1,30 @@
+# 2024 NVIDIA CORPORATION & AFFILIATES
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ofed-modules-blacklist
+  namespace: {{ .RuntimeSpec.Namespace }}
+data:
+  blacklist-ofed-modules.conf: |-
+    # blacklist ofed-related modules on host to prevent inbox or host OFED driver loading
+    blacklist mlx5_core
+    blacklist mlx5_ib
+    blacklist ib_umad
+    blacklist ib_uverbs
+    blacklist ib_ipoib
+    blacklist rdma_cm
+    blacklist rdma_ucm
+    blacklist ib_core
+    blacklist ib_cm
diff --git a/...tate-ofed-driver/0050_ofed-driver-ds.yaml → ...tate-ofed-driver/0060_ofed-driver-ds.yaml b/...tate-ofed-driver/0050_ofed-driver-ds.yaml → ...tate-ofed-driver/0060_ofed-driver-ds.yaml
@@ -81,6 +81,13 @@ spec:
         - image: {{ .RuntimeSpec.MOFEDImageName }}
           imagePullPolicy: IfNotPresent
           name: mofed-container
+          lifecycle:
+            postStart:
+              exec:
+                command: [ "cp", "/opt/blacklist-ofed-modules.conf", "/host/etc/modprobe.d/" ]
+            preStop:
+              exec:
+                command: [ "rm", "/host/etc/modprobe.d/blacklist-ofed-modules.conf" ]
           securityContext:
             privileged: true
             seLinuxOptions:
@@ -118,6 +125,9 @@ spec:
               mountPath: /mnt/drivers-inventory
             - name: host-ib-core
               mountPath: /etc/modprobe.d/ib_core.conf
+            - name: modules-blacklist
+              mountPath: /opt/blacklist-ofed-modules.conf
+              subPath: blacklist-ofed-modules.conf
             {{- if.AdditionalVolumeMounts.VolumeMounts }}
             {{- range .AdditionalVolumeMounts.VolumeMounts }}
             - name: {{ .Name }}
@@ -245,6 +255,9 @@ spec:
           hostPath:
             path: /etc/modprobe.d/ib_core.conf
             type: FileOrCreate
+        - name: modules-blacklist
+          configMap:
+            name: ofed-modules-blacklist
         {{- range .AdditionalVolumeMounts.Volumes }}
         - name: {{ .Name }}
           {{- if and .ConfigMap .ConfigMap.Items }}

diff --git a/pkg/state/state_ofed_test.go b/pkg/state/state_ofed_test.go
@@ -307,8 +307,8 @@ var _ = Describe("MOFED state test", func() {
 			catalog.Add(InfoTypeDocaDriverImage, &dummyOfedImageProvider{tagExists: true})
 			objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger)
 			Expect(err).NotTo(HaveOccurred())
-			// Expect 5 objects: 1 DS per pool, Service Account, Role, RoleBinding
-			Expect(len(objs)).To(Equal(5))
+			// Expect 5 objects: 1 DS per pool, Service Account, Role, RoleBinding, ConfigMap
+			Expect(len(objs)).To(Equal(6))
 			By("Verify DaemonSets NodeSelector")
 			for _, obj := range objs {
 				if obj.GetKind() != "DaemonSet" {
@@ -365,8 +365,8 @@ var _ = Describe("MOFED state test", func() {
 			catalog.Add(InfoTypeDocaDriverImage, &dummyOfedImageProvider{tagExists: false})
 			objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger)
 			Expect(err).NotTo(HaveOccurred())
-			// Expect 5 objects: 1 DS per pool, Service Account, Role, RoleBinding
-			Expect(len(objs)).To(Equal(4))
+			// Expect 5 objects: 1 DS per pool, Service Account, Role, RoleBinding, ConfigMap
+			Expect(len(objs)).To(Equal(5))
 			By("Verify Subscription mounts")
 			for _, obj := range objs {
 				if obj.GetKind() != "DaemonSet" {
@@ -470,8 +470,8 @@ var _ = Describe("MOFED state test", func() {
 			objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger)
 			Expect(err).NotTo(HaveOccurred())
 			// Expect 6 object due to OpenShift: DaemonSet, Service Account, ClusterRole, ClusterRoleBinding
-			// Role, RoleBinding
-			Expect(len(objs)).To(Equal(6))
+			// Role, RoleBinding, ConfigMap
+			Expect(len(objs)).To(Equal(7))
 			By("Verify DaemonSet with DTK")
 			for _, obj := range objs {
 				if obj.GetKind() != "DaemonSet" {
@@ -539,8 +539,8 @@ var _ = Describe("MOFED state test", func() {
 			objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger)
 			Expect(err).NotTo(HaveOccurred())
 			By("Verify image is not precompiled format")
-			// Expect 4 objects: DS , Service Account, Role, RoleBinding
-			Expect(len(objs)).To(Equal(4))
+			// Expect 4 objects: DS , Service Account, Role, RoleBinding, ConfigMap
+			Expect(len(objs)).To(Equal(5))
 			for _, obj := range objs {
 				if obj.GetKind() != "DaemonSet" {
 					continue
@@ -577,8 +577,8 @@ var _ = Describe("MOFED state test", func() {
 			objs, err := ofedState.GetManifestObjects(ctx, cr, catalog, testLogger)
 			Expect(err).NotTo(HaveOccurred())
 			By("Verify image is not precompiled format")
-			// Expect 4 objects: DS , Service Account, Role, RoleBinding
-			Expect(len(objs)).To(Equal(4))
+			// Expect 4 objects: DS , Service Account, Role, RoleBinding, ConfigMap
+			Expect(len(objs)).To(Equal(5))
 			for _, obj := range objs {
 				if obj.GetKind() != "DaemonSet" {
 					continue