python: updates to sdk to support range experiment

Problem: we need to update the python sdk to support range Solution: update the protos and the config. This also comments out a lot of verbosity (for now). I would like to have a more hardened logging approach that is tied to a command line flag / config parameter / envar or similar. Signed-off-by: vsoch <[email protected]>
converged-computing · Mar 31, 2024 · 37f0dbd · 37f0dbd
1 parent efa6e6c
commit 37f0dbd
Show file tree

Hide file tree

Showing 9 changed files with 58 additions and 25 deletions.
diff --git a/README.md b/README.md
@@ -13,6 +13,7 @@ For more information:
 
 ## TODO
 
+- nice logger with actual levels that I like
 - subsystems
   - make also a function to delete subsystems
 - ephemeral case - actual nodes don't exist, but instead rules for requests and limits. Need to develop this and means to authenticate to use it.

diff --git a/docs/examples/match-algorithms/range/rainbow-config.yaml b/docs/examples/match-algorithms/range/rainbow-config.yaml
@@ -8,7 +8,7 @@ scheduler:
             name: match
 cluster:
     name: spack-builder
-    secret: 85e59eea-c427-4f55-9668-4ed418de9be8
+    secret: bd72a288-cc3d-4659-910b-d665fd95f1a3
 graphdatabase:
     name: memory
     host: 127.0.0.1:50051

diff --git a/plugins/algorithms/range/range.go b/plugins/algorithms/range/range.go
@@ -74,7 +74,7 @@ func (req *RangeRequest) Satisfies(value string) (bool, error) {
 	}
 	if req.Min != "" {
 		// Is the version provided greater than the min requested?
-		c, err := semver.NewConstraint(fmt.Sprintf("> %s", req.Min))
+		c, err := semver.NewConstraint(fmt.Sprintf(">= %s", req.Min))
 		if err != nil {
 			// fmt.Printf("      => Error parsing min constraint %s\n", err)
 			return false, err
@@ -89,7 +89,7 @@ func (req *RangeRequest) Satisfies(value string) (bool, error) {
 	}
 	if req.Max != "" {
 		// Is the version provided less than the max requested?
-		c, err := semver.NewConstraint(fmt.Sprintf("< %s", req.Max))
+		c, err := semver.NewConstraint(fmt.Sprintf("<= %s", req.Max))
 		if err != nil {
 			// fmt.Printf("      => Error parsing max constraint %s\n", err)
 			return false, err

diff --git a/plugins/backends/memory/dfs.go b/plugins/backends/memory/dfs.go
@@ -54,7 +54,7 @@ func (g *ClusterGraph) DFSForMatch(
 	}
 
 	// Get the summary metrics for the subsystem
-	fmt.Println(ss.Metrics.ResourceCounts)
+	// fmt.Println(ss.Metrics.ResourceCounts)
 
 	isMatch := true
 	for resourceType, needed := range totals {
@@ -90,9 +90,7 @@ func (g *ClusterGraph) depthFirstSearch(
 ) (bool, error) {
 
 	// Note that in the experimental version we have one task and thus one slot
-	if !g.quiet {
-		fmt.Printf("  🎰️ Slots that need to be satisfied with matcher %s\n", matcher.Name())
-	}
+	//	fmt.Printf("  🎰️ Slots that need to be satisfied with matcher %s\n", matcher.Name())
 	slots := map[string]*v1.Task{}
 
 	// If a slot isn't defined for the task, assume the slot is at the top level
@@ -106,9 +104,7 @@ func (g *ClusterGraph) depthFirstSearch(
 	// If we don't have jobspec.Task.Resources, no slot to search for.
 	// Return early based on top level counts
 	if len(jobspec.Task.Resources) == 0 {
-		if !g.quiet {
-			fmt.Printf("  🎰️ No resources defined, top level counts satisfied so cluster is match\n")
-		}
+		// fmt.Printf("  🎰️ No resources defined, top level counts satisfied so cluster is match\n")
 		return true, nil
 	}
 
@@ -143,22 +139,18 @@ func (g *ClusterGraph) depthFirstSearch(
 		}
 
 		// Subsystem edges are here, separate from dominant ones (so search is smaller)
-		for sName, edges := range vtx.Subsystems {
+		for _, edges := range vtx.Subsystems {
 			// fmt.Printf("      => Searching for %s and resource type %s in subsystem %v with %d subsystem edges\n", lookingFor, resource.Type, sName, len(edges))
 
 			for _, child := range edges {
-				if !g.quiet {
-					fmt.Printf("         Found subsystem edge %s with type %s\n", sName, child.Vertex.Type)
-				}
+				// fmt.Printf("         Found subsystem edge %s with type %s\n", sName, child.Vertex.Type)
 				// Check if the subsystem edge satisfies the needs of the slot
 				// This will update the slotNeeds.Satisfied
 				matcher.CheckSubsystemEdge(slotNeeds, child, vtx)
 
 				// Return early if minimum needs are satsified
 				if slotNeeds.Satisfied {
-					if !g.quiet {
-						fmt.Printf("         Minimum slot needs are satisfied at %s for %s at %s, returning early.\n", vtx.Type, child.Subsystem, child.Vertex.Type)
-					}
+					// fmt.Printf("         Minimum slot needs are satisfied at %s for %s at %s, returning early.\n", vtx.Type, child.Subsystem, child.Vertex.Type)
 					return slotsFound + vtx.Size
 				}
 			}

diff --git a/python/v1/rainbow/backends/memory.py b/python/v1/rainbow/backends/memory.py
@@ -15,13 +15,13 @@ class MemoryBackend(GraphBackend):
     This graph database backend is primarily for development.
     """
 
-    def satisfies(self, jobspec):
+    def satisfies(self, jobspec, matcher="match"):
         """
         Determine if a jobspec can be satisfied by the graph.
         """
         # Prepare a satisfy request with the jobspec
         # TODO if auth is in the graph, that needs to be done here too
-        request = memory_pb2.SatisfyRequest(payload=jobspec.to_str())
+        request = memory_pb2.SatisfyRequest(payload=jobspec.to_str(), matcher=matcher)
 
         # Host should be set from the database_options from the client
         with grpc.insecure_channel(self.host) as channel:

diff --git a/python/v1/rainbow/client.py b/python/v1/rainbow/client.py
@@ -161,7 +161,7 @@ def submit_jobspec(self, jobspec):
         it custom with your own special logic.
         """
         # Ask the database backend if our jobspec can be satisfied
-        response = self.backend.satisfies(jobspec)
+        response = self.backend.satisfies(jobspec, self.cfg.match_algorithm)
         matches = response.clusters
 
         # No matches?

diff --git a/python/v1/rainbow/config.py b/python/v1/rainbow/config.py
@@ -11,7 +11,7 @@ def new_rainbow_config(host, cluster, secret, scheduler_name="rainbow"):
         "scheduler": {
             "name": scheduler_name,
             "secret": secret,
-            "algorithm": {"name": "random"},
+            "algorithms": {"selection": {"name": "random"}, "match": {"name": "match"}},
         },
         "cluster": {
             "name": cluster,
@@ -52,6 +52,34 @@ def cfg(self):
             self.load()
         return self._cfg
 
+    @property
+    def match_algorithm(self):
+        """
+        Get the match algorithm
+        """
+        matcher = self._cfg.get("scheduler", {}).get("algorithms", {}).get("match", {}).get("name")
+        if not matcher:
+            matcher = "match"
+        return matcher
+
+    def set_match_algorithm(self, name):
+        """
+        Get the match algorithm
+        """
+        self._set_algorithm("match", name)
+
+    def set_selection_algorithm(self, name):
+        """
+        Get the match algorithm
+        """
+        self._set_algorithm("selection", name)
+
+    def _set_algorithm(self, typ, name):
+        """
+        Get the match algorithm
+        """
+        self._cfg["scheduler"]["algorithms"][typ]["name"] = name
+
     def load(self, config_file=None):
         """
         Load a rainbow config

diff --git a/python/v1/rainbow/schema.py b/python/v1/rainbow/schema.py
@@ -12,11 +12,23 @@
             "properties": {
                 "name": {"type": "string"},
                 "secret": {"type": "string"},
-                "algorithm": {
+                "algorithms": {
                     "type": "object",
                     "properties": {
-                        "name": {"type": "string"},
-                        "options": {"type": "object"},
+                        "selection": {
+                            "type": "object",
+                            "properties": {
+                                "name": {"type": "string"},
+                                "options": {"type": "object"},
+                            },
+                        },
+                        "match": {
+                            "type": "object",
+                            "properties": {
+                                "name": {"type": "string"},
+                                "options": {"type": "object"},
+                            },
+                        },
                     },
                 },
                 "user": {"type": "object"},

diff --git a/python/v1/setup.py b/python/v1/setup.py
@@ -18,7 +18,7 @@
 if __name__ == "__main__":
     setup(
         name="rainbow-scheduler",
-        version="0.0.14rc1",
+        version="0.0.14",
         author="Vanessasaurus",
         author_email="[email protected]",
         maintainer="Vanessasaurus",