python: updates to sdk to support range experiment

cmd/server/server.go

@@ -6,6 +6,7 @@ import (
 	"log"
 
 	"github.com/converged-computing/rainbow/pkg/config"
+	rlog "github.com/converged-computing/rainbow/pkg/logger"
 	"github.com/converged-computing/rainbow/pkg/server"
 	"github.com/converged-computing/rainbow/pkg/types"
 
@@ -17,16 +18,19 @@ import (
 )
 
 var (
-	host        string
-	name        = "rainbow"
-	sqliteFile  = "rainbow.db"
-	configFile  = ""
-	matchAlgo   = "match"
-	selectAlgo  = "random"
-	database    = ""
-	cleanup     = false
-	secret      = "chocolate-cookies"
-	globalToken = ""
+	host string
+
+	// default logging level of warning (none, info, warning)
+	loggingLevel = 3
+	name         = "rainbow"
+	sqliteFile   = "rainbow.db"
+	configFile   = ""
+	matchAlgo    = "match"
+	selectAlgo   = "random"
+	database     = ""
+	cleanup      = false
+	secret       = "chocolate-cookies"
+	globalToken  = ""
 )
 
 func main() {
@@ -39,9 +43,15 @@ func main() {
 	flag.StringVar(&selectAlgo, "select-algorithm", selectAlgo, "selection algorithm for final cluster selection (defaults to random)")
 	flag.StringVar(&matchAlgo, "match-algorithm", matchAlgo, "match algorithm for graph (defaults to random)")
 	flag.StringVar(&configFile, "config", configFile, "rainbow config file")
+	flag.IntVar(&loggingLevel, "loglevel", loggingLevel, "rainbow logging level (0 to 5)")
 	flag.BoolVar(&cleanup, "cleanup", cleanup, "cleanup previous sqlite database (default: false)")
 	flag.Parse()
 
+	// If the logging level isn't the default, set it
+	if loggingLevel != rlog.DefaultLevel {
+		rlog.SetLevel(loggingLevel)
+	}
+
 	// Load (or generate a default)  config file here, if provided
 	cfg, err := config.NewRainbowClientConfig(configFile, name, secret, database, selectAlgo, matchAlgo)
 	if err != nil {

docs/algorithms.md

@@ -121,7 +121,7 @@ I understand this is likely not perfect for what everyone wants, but I believe i
 
 ### Match
 
-The expliciy "match" type is going to look exactly at the type of a subsystem node, and return true (match) if it matches what the subsystem needs. For example, given this task:
+The expliciy "match" type is going to look exactly at some exact value for a field in the metadata. It will return true (match) if it matches what the subsystem needs. For example, given this task:
 
 ```yaml
 task:
@@ -133,10 +133,11 @@ task:
   resources:
     io:
       match:
-      - type: shm
+      - field: type
+        value: shm
 ```
 
-We would look for a node of type "shm" in the io subsystem that is directly attached (an edge) to a node in the dominant subsystem graph.
+We would look for a node with field "type" and value "shm" in the io subsystem that is directly attached (an edge) to a node in the dominant subsystem graph.
 
 ### Range
 

docs/examples/match-algorithms/range/jobspec-invalid-range.yaml

@@ -11,7 +11,7 @@ resources:
       type: core
 task:
   command:
-  - ior
+  - spack
   slot: default
   count:
     per_slot: 1

docs/examples/match-algorithms/range/rainbow-config.yaml

@@ -8,7 +8,7 @@ scheduler:
             name: match
 cluster:
     name: spack-builder
-    secret: 85e59eea-c427-4f55-9668-4ed418de9be8
+    secret: 37e5b798-189f-4c38-bc1c-0a14877acbcf
 graphdatabase:
     name: memory
     host: 127.0.0.1:50051

docs/examples/scheduler/jobspec-io.yaml

@@ -18,4 +18,5 @@ task:
   resources:
     io:
       match:
-      - type: shm
+      - field: type
+        value: shm

docs/examples/scheduler/rainbow-config.yaml

@@ -8,7 +8,7 @@ scheduler:
             name: match
 cluster:
     name: keebler
-    secret: 3994c1e7-9cc7-4b81-ab75-3b00128eda16
+    secret: 79643f8e-6408-4cd1-bd1a-76aa499d5864
 graphdatabase:
     name: memory
     host: 127.0.0.1:50051

go.mod

@@ -7,6 +7,7 @@ require (
 	github.com/akamensky/argparse v1.4.0
 	github.com/compspec/jobspec-go v0.0.0-20240319000127-8020a01a65da
 	github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe
+	github.com/fatih/color v1.16.0
 	github.com/google/uuid v1.6.0
 	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/pkg/errors v0.9.1
@@ -17,6 +18,8 @@ require (
 
 require (
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
 	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect

go.sum

@@ -6,6 +6,8 @@ github.com/compspec/jobspec-go v0.0.0-20240319000127-8020a01a65da h1:Uvfk4IgWMIi
 github.com/compspec/jobspec-go v0.0.0-20240319000127-8020a01a65da/go.mod h1:BaJyxaOhESe2DD4lqBdwTEWOw0TaTZVJGPrFh6KyXQM=
 github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe h1:Tk//RW3uKn4A7N8gpHRXs+ZGlR7Fxkwh+4/Iml0GBV4=
 github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk=
+github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
+github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
 github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
@@ -14,6 +16,11 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -22,6 +29,8 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng
 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
 golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
 golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=

hack/run-range-register.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+go run cmd/rainbow/rainbow.go config init --cluster-name spack-builder --config-path ./docs/examples/match-algorithms/range/rainbow-config.yaml --match-algorithm range
+
+# Register your nodes
+go run cmd/rainbow/rainbow.go register cluster --cluster-name spack-builder --nodes-json ./docs/examples/match-algorithms/range/cluster-nodes.json --config-path ./docs/examples/match-algorithms/range/rainbow-config.yaml --save
+
+# Register the subsystem
+go run cmd/rainbow/rainbow.go register subsystem  --subsystem spack --nodes-json ./docs/examples/match-algorithms/range/spack-subsystem.json --config-path ./docs/examples/match-algorithms/range/rainbow-config.yaml
+
+# Submit a job that asked for a valid range
+go run ./cmd/rainbow/rainbow.go submit --config-path ./docs/examples/match-algorithms/range/rainbow-config.yaml --jobspec ./docs/examples/match-algorithms/range/jobspec-valid-range.yaml --match-algorithm range

pkg/logger/logger.go

@@ -0,0 +1,178 @@
+package logger
+
+// TODO would like to use slog when we can use go 1.21!
+
+import (
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/fatih/color"
+)
+
+const (
+	LevelNone = iota
+	LevelInfo
+	LevelWarning
+	LevelError
+	LevelVerbose
+	LevelDebug
+)
+
+var (
+	DefaultLevel = 3
+	logger       *RainbowLogger
+)
+
+type RainbowLogger struct {
+	level    int
+	Filename string
+	handle   *os.File
+}
+
+// Start opens a file handle, if it's desired to write to file
+func (l *RainbowLogger) Start() (*log.Logger, error) {
+	f, err := os.OpenFile(l.Filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm)
+	if err != nil {
+		return nil, err
+	}
+	logger := log.New(f, "", 0)
+	l.handle = f
+	return logger, nil
+}
+
+// Stop closes the file handle, if defined
+func (l *RainbowLogger) Stop() error {
+	if l.handle != nil {
+		return l.handle.Close()
+	}
+	return nil
+}
+
+// Logging functions with formatting
+func Infof(message ...any) error {
+	return logger.logFormat(LevelInfo, message...)
+}
+
+func Errorf(message ...any) error {
+	color.Set(color.FgRed)
+	err := logger.logFormat(LevelError, message...)
+	color.Unset()
+	return err
+}
+func Debugf(message ...any) error {
+	color.Set(color.FgBlue)
+	err := logger.logFormat(LevelDebug, message...)
+	color.Unset()
+	return err
+
+}
+func Verbosef(message ...any) error {
+	color.Set(color.FgMagenta)
+	err := logger.logFormat(LevelVerbose, message...)
+	color.Unset()
+	return err
+
+}
+func Warningf(message ...any) error {
+	color.Set(color.FgYellow)
+	err := logger.logFormat(LevelWarning, message...)
+	color.Unset()
+	return err
+}
+
+// And without!
+func Info(message string) error {
+	return logger.log(LevelInfo, message)
+}
+func Error(message string) error {
+	color.Set(color.FgRed)
+	err := logger.log(LevelError, message)
+	color.Unset()
+	return err
+
+}
+func Debug(message string) error {
+	color.Set(color.FgBlue)
+	err := logger.log(LevelDebug, message)
+	color.Unset()
+	return err
+}
+func Verbose(message string) error {
+	color.Set(color.FgMagenta)
+	err := logger.log(LevelVerbose, message)
+	color.Unset()
+	return err
+}
+func Warning(message string) error {
+	color.Set(color.FgYellow)
+	err := logger.log(LevelWarning, message)
+	color.Unset()
+	return err
+}
+
+// log prints (without formatting) to the log
+func (l *RainbowLogger) log(level int, message string) error {
+	if l.Filename != "" {
+		l.logToFile(level, message)
+	}
+	if level >= l.level {
+		fmt.Println(message)
+	}
+	return nil
+}
+
+// logFormat is the shared class function for actually printing to the log
+func (l *RainbowLogger) logFormat(level int, message ...any) error {
+	if l.Filename != "" {
+		l.logFormatToFile(level, message)
+	}
+	// Otherwise just print! Simple and dumb
+	prolog := message[0].(string)
+	rest := message[1:]
+	if level <= l.level {
+		fmt.Printf(prolog, rest...)
+	}
+	return nil
+}
+
+// logFormatToFile writes to file if the rainbow logger is set to do that
+func (l *RainbowLogger) logFormatToFile(level int, message ...any) error {
+	logger, err := l.Start()
+	if err != nil {
+		return err
+	}
+	// Assume the prolog (to be formatted) is at index 0
+	prolog := message[0].(string)
+	rest := message[1:]
+	if level <= l.level {
+		logger.Printf(prolog, rest...)
+	}
+	return l.Stop()
+}
+
+// logToFile writes to file if the rainbow logger is set to do that
+func (l *RainbowLogger) logToFile(level int, message string) error {
+	logger, err := l.Start()
+	if err != nil {
+		return err
+	}
+	if level <= l.level {
+		logger.Println(message)
+	}
+	return l.Stop()
+}
+
+// WriteToFile will set the global level and filename
+func WriteToFile(filename string) {
+	logger.Filename = filename
+}
+
+// SetLevel sets the global logging level
+func SetLevel(level int) {
+	logger.level = level
+}
+
+func init() {
+	logger = &RainbowLogger{level: LevelWarning}
+}