custom: add support for custom container (#84)

* custom: add support for custom container We should be able to support custom containers, and configuration of addons to them. I am not liking the design to have addons defined in parallel, and want to refactor so they are part of the metric. I am also wondering if the metrics themselves are more akin to apps. I have not looked at this project in a bit and need to think about it. Signed-off-by: vsoch <[email protected]>
converged-computing · Sep 24, 2024 · 8f15755 · 8f15755
1 parent 8835f15
commit 8f15755
Show file tree

Hide file tree

Showing 23 changed files with 340 additions and 76 deletions.
diff --git a/.github/workflows/build-deploy.yaml b/.github/workflows/build-deploy.yaml
@@ -18,7 +18,7 @@ jobs:
       uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.22
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
       uses: docker/login-action@v2
@@ -48,7 +48,7 @@ jobs:
       uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.22
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
       uses: docker/login-action@v2

diff --git a/.github/workflows/helm.yaml b/.github/workflows/helm.yaml
@@ -16,7 +16,7 @@ jobs:
       uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.18.1
+        go-version: ^1.22
     - name: GHCR Login
       if: (github.event_name != 'pull_request')
       uses: docker/login-action@v2

diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -34,7 +34,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v3
       with:
-        go-version: ^1.20
+        go-version: ^1.22
     - name: fmt check
       run: make fmt
 
@@ -88,7 +88,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v3
       with:
-        go-version: ^1.20
+        go-version: ^1.22
 
     - name: Start minikube
       uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9

diff --git a/.github/workflows/python.yaml b/.github/workflows/python.yaml
@@ -34,7 +34,7 @@ jobs:
     - name: Setup Go
       uses: actions/setup-go@v3
       with:
-        go-version: ^1.20
+        go-version: ^1.22
 
     - name: Start minikube
       uses: medyagh/setup-minikube@697f2b7aaed5f70bf2a94ee21a4ec3dde7b12f92 # v0.0.9

diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -21,7 +21,7 @@ jobs:
         echo "tag=${{ inputs.release_tag }}" >> ${GITHUB_ENV}
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.20
+        go-version: ^1.22
     - name: GHCR Login
       uses: docker/login-action@v2
       with:
@@ -51,7 +51,7 @@ jobs:
       uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.20
+        go-version: ^1.22
     - name: Set tag
       run: |
         echo "Tag for release is ${{ inputs.release_tag }}"
@@ -86,7 +86,7 @@ jobs:
       uses: actions/checkout@v3
     - uses: actions/setup-go@v3
       with:
-        go-version: ^1.20
+        go-version: ^1.22
     - name: Set tag
       run: |
         echo "Tag for release is ${{ inputs.release_tag }}"

diff --git a/Dockerfile b/Dockerfile
@@ -1,5 +1,5 @@
 # Build the manager binary
-FROM golang:1.20 as builder
+FROM golang:1.22 as builder
 ARG TARGETOS
 ARG TARGETARCH
 

diff --git a/Makefile b/Makefile
@@ -173,6 +173,19 @@ deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
 	$(KUSTOMIZE) build config/default | kubectl delete --ignore-not-found=$(ignore-not-found) -f -
 
+
+.PHONY: test-deploy
+test-deploy: manifests kustomize
+	docker build --no-cache -t ${DEVIMG} .
+	docker push ${DEVIMG}
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${DEVIMG}
+	$(KUSTOMIZE) build config/default > examples/dist/metrics-operator-dev.yaml
+
+.PHONY: test-deploy-recreate
+test-deploy-recreate: test-deploy
+	kubectl delete -f ./examples/dist/metrics-operator-dev.yaml || echo "Already deleted"
+	kubectl apply -f ./examples/dist/metrics-operator-dev.yaml
+
 ##@ Build Dependencies
 
 ## Location to install dependencies to
@@ -187,7 +200,7 @@ ENVTEST ?= $(LOCALBIN)/setup-envtest
 
 ## Tool Versions
 KUSTOMIZE_VERSION ?= v3.8.7
-CONTROLLER_TOOLS_VERSION ?= v0.11.1
+CONTROLLER_TOOLS_VERSION ?= v0.14.0
 
 KUSTOMIZE_INSTALL_SCRIPT ?= "https://raw.githubusercontent.com/kubernetes-sigs/kustomize/master/hack/install_kustomize.sh"
 .PHONY: kustomize
@@ -205,6 +218,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 	test -s $(LOCALBIN)/controller-gen && $(LOCALBIN)/controller-gen --version | grep -q $(CONTROLLER_TOOLS_VERSION) || \
 	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
 
+
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)

diff --git a/api/v1alpha2/zz_generated.deepcopy.go b/api/v1alpha2/zz_generated.deepcopy.go
diff --git a/config/crd/bases/flux-framework.org_metricsets.yaml b/config/crd/bases/flux-framework.org_metricsets.yaml
@@ -3,8 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.11.1
-  creationTimestamp: null
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: metricsets.flux-framework.org
 spec:
   group: flux-framework.org
@@ -21,14 +20,19 @@ spec:
         description: MetricSet is the Schema for the metrics API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -37,21 +41,23 @@ spec:
             properties:
               deadlineSeconds:
                 default: 31500000
-                description: Should the job be limited to a particular number of seconds?
+                description: |-
+                  Should the job be limited to a particular number of seconds?
                   Approximately one year. This cannot be zero or job won't start
                 format: int64
                 type: integer
               dontSetFQDN:
                 description: Don't set JobSet FQDN
                 type: boolean
               logging:
-                description: Logging spec, preparing for other kinds of logging Right
-                  now we just include an interactive option
+                description: |-
+                  Logging spec, preparing for other kinds of logging
+                  Right now we just include an interactive option
                 properties:
                   interactive:
-                    description: Don't allow the application, metric, or storage test
-                      to finish This adds sleep infinity at the end to allow for interactive
-                      mode.
+                    description: |-
+                      Don't allow the application, metric, or storage test to finish
+                      This adds sleep infinity at the end to allow for interactive mode.
                     type: boolean
                 type: object
               metrics:
@@ -60,15 +66,15 @@ spec:
                 items:
                   properties:
                     addons:
-                      description: A Metric addon can be storage (volume) or an application,
-                        It's an additional entity that can customize a replicated
-                        job, either adding assets / features or entire containers
-                        to the pod
+                      description: |-
+                        A Metric addon can be storage (volume) or an application,
+                        It's an additional entity that can customize a replicated job,
+                        either adding assets / features or entire containers to the pod
                       items:
-                        description: 'A Metric addon is an interface that exposes
-                          extra volumes for a metric. Examples include: A storage
-                          volume to be mounted on one or more of the replicated jobs
-                          A single application container.'
+                        description: |-
+                          A Metric addon is an interface that exposes extra volumes for a metric. Examples include:
+                          A storage volume to be mounted on one or more of the replicated jobs
+                          A single application container.
                         properties:
                           listOptions:
                             additionalProperties:
@@ -129,7 +135,9 @@ spec:
                           - type: string
                           x-kubernetes-int-or-string: true
                         type: array
-                      description: Metric List Options Metric specific options
+                      description: |-
+                        Metric List Options
+                        Metric specific options
                       type: object
                     mapOptions:
                       additionalProperties:
@@ -149,7 +157,9 @@ spec:
                         - type: integer
                         - type: string
                         x-kubernetes-int-or-string: true
-                      description: Metric Options Metric specific options
+                      description: |-
+                        Metric Options
+                        Metric specific options
                       type: object
                     resources:
                       description: Resources include limits and requests for the metric

diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml
@@ -2,7 +2,6 @@
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: manager-role
 rules:
 - apiGroups:

diff --git a/controllers/metric/metric_controller.go b/controllers/metric/metric_controller.go
@@ -129,7 +129,7 @@ func (r *MetricSetReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	// Ensure the metricset is mapped to a JobSet. For design:
 	// 1. If an application is provided, we pair the application at some scale with each metric as a contaienr
-	// 2. If storage is provided, we create the volumes for the metric containers
+	// 2. If storage or other addons are provided, we create the volumes for the metric containers
 	result, err := r.ensureMetricSet(ctx, &spec, &set)
 	if err != nil {
 		r.Log.Error(err, "🟥️ Issue ensuring metric set")

diff --git a/docs/_static/data/metrics.json b/docs/_static/data/metrics.json
@@ -20,6 +20,13 @@
   "image": "ghcr.io/converged-computing/metric-cabanapic:latest",
   "url": "https://github.com/ECP-copa/CabanaPIC"
  },
+ {
+  "name": "app-custom",
+  "description": "Provide a custom application for MPI trace",
+  "family": "proxyapp",
+  "image": "",
+  "url": "https://converged-computing.github.io/metrics-operator"
+ },
  {
   "name": "app-hpl",
   "description": "High-Performance Linpack (HPL)",

diff --git a/docs/getting_started/addons.md b/docs/getting_started/addons.md
@@ -270,7 +270,7 @@ wrapper to the actual executable.
 
 ### perf-mpitrace
 
- - *[perf-mpitrace](https://github.com/converged-computing/metrics-operator/tree/main/examples/addons/perf-mpitrace)*
+ - *[perf-mpitrace](https://github.com/converged-computing/metrics-operator/tree/main/examples/addons/mpitrace-lammps)*
 
 This metric provides [mpitrace](https://github.com/IBM/mpitrace) to wrap an MPI application. The setup is the same as hpctoolkit, and we
 currently only provide a rocky base (please let us know if you need another). It works by way of wrapping the mpirun command with `LD_PRELOAD`.

diff --git a/docs/getting_started/metrics.md b/docs/getting_started/metrics.md
@@ -251,6 +251,49 @@ Here are some useful resources for the benchmarks:
  - [HPC Council](https://hpcadvisorycouncil.atlassian.net/wiki/spaces/HPCWORKS/pages/1284538459/OSU+Benchmark+Tuning+for+2nd+Gen+AMD+EPYC+using+HDR+InfiniBand+over+HPC-X+MPI)
  - [AWS Tutorials](https://www.hpcworkshops.com/08-efa/04-complie-run-osu.html)
 
+### app-custom
+
+A custom application can support any application to be used as a metric app. For the following parameters, "command" and "container" are required.
+
+| Name | Description | Option Key | Type | Default |
+|-----|-------------|------------|------|---------|
+| command | The full mpirun command | options->command |string | unset |
+| workdir | The working directory for the command | options->workdir | string | unset |
+| soleTenancy | require each pod to have sole tenancy | command->soleTenancy | string | "false" |
+
+As an example, here is running mpitrace (an addon) with a custom container.
+
+```yaml
+apiVersion: flux-framework.org/v1alpha2
+kind: MetricSet
+metadata:
+  labels:
+    app.kubernetes.io/name: metricset
+    app.kubernetes.io/instance: metricset-sample
+  name: metricset-sample
+spec:
+  # Number of pods for lammps (one launcher, the rest workers)
+  pods: 4
+  metrics:
+   - name: app-custom
+     image: ghcr.io/converged-computing/<your-container>
+     options:
+       command: mpirun --hostfile ./hostlist.txt -mca orte_keep_fqdn_hostnames t -np 4 --map-by socket <app> <options>
+       workdir: <workdir>
+
+     # Add on hpctoolkit, will mount a volume and wrap lammps
+     addons:
+       - name: perf-mpitrace
+         options:
+           mount: /opt/mnt
+           image: ghcr.io/converged-computing/metric-mpitrace:ubuntu-jammy
+           workdir: <workdir>
+           # this is the target of the replicated job "l" means launcher
+           target: l          
+           # This is the target container, with full name "launcher"
+           containerTarget: launcher
+```
+
 ### app-lammps
 
  - *[app-lammps](https://github.com/converged-computing/metrics-operator/tree/main/examples/tests/app-lammps)*

diff --git a/examples/addons/mpitrace-lammps/metrics-rocky.yaml b/examples/addons/mpitrace-lammps/metrics-rocky.yaml
@@ -22,7 +22,6 @@ spec:
        command: /opt/intel/mpi/2021.8.0/bin/mpirun --hostfile ./hostlist.txt -np 4 --map-by socket lmp -v x 2 -v y 2 -v z 2 -in in.reaxc.hns -nocite
        workdir: /opt/lammps/examples/reaxff/HNS
 
-     # Add on hpctoolkit, will mount a volume and wrap lammps
      addons:
        - name: perf-mpitrace
          options: