Skip to content

Commit

Permalink
feat: custom rainbow logger and updates to rainbow python
Browse files Browse the repository at this point in the history
Signed-off-by: vsoch <[email protected]>
  • Loading branch information
vsoch committed Apr 1, 2024
1 parent 37f0dbd commit ef4376e
Show file tree
Hide file tree
Showing 26 changed files with 443 additions and 153 deletions.
1 change: 0 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,6 @@ For more information:

## TODO

- nice logger with actual levels that I like
- subsystems
- make also a function to delete subsystems
- ephemeral case - actual nodes don't exist, but instead rules for requests and limits. Need to develop this and means to authenticate to use it.
Expand Down
30 changes: 20 additions & 10 deletions cmd/server/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"log"

"github.com/converged-computing/rainbow/pkg/config"
rlog "github.com/converged-computing/rainbow/pkg/logger"
"github.com/converged-computing/rainbow/pkg/server"
"github.com/converged-computing/rainbow/pkg/types"

Expand All @@ -17,16 +18,19 @@ import (
)

var (
host string
name = "rainbow"
sqliteFile = "rainbow.db"
configFile = ""
matchAlgo = "match"
selectAlgo = "random"
database = ""
cleanup = false
secret = "chocolate-cookies"
globalToken = ""
host string

// default logging level of warning (none, info, warning)
loggingLevel = 3
name = "rainbow"
sqliteFile = "rainbow.db"
configFile = ""
matchAlgo = "match"
selectAlgo = "random"
database = ""
cleanup = false
secret = "chocolate-cookies"
globalToken = ""
)

func main() {
Expand All @@ -39,9 +43,15 @@ func main() {
flag.StringVar(&selectAlgo, "select-algorithm", selectAlgo, "selection algorithm for final cluster selection (defaults to random)")
flag.StringVar(&matchAlgo, "match-algorithm", matchAlgo, "match algorithm for graph (defaults to random)")
flag.StringVar(&configFile, "config", configFile, "rainbow config file")
flag.IntVar(&loggingLevel, "loglevel", loggingLevel, "rainbow logging level (0 to 5)")
flag.BoolVar(&cleanup, "cleanup", cleanup, "cleanup previous sqlite database (default: false)")
flag.Parse()

// If the logging level isn't the default, set it
if loggingLevel != rlog.DefaultLevel {
rlog.SetLevel(loggingLevel)
}

// Load (or generate a default) config file here, if provided
cfg, err := config.NewRainbowClientConfig(configFile, name, secret, database, selectAlgo, matchAlgo)
if err != nil {
Expand Down
7 changes: 4 additions & 3 deletions docs/algorithms.md
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ I understand this is likely not perfect for what everyone wants, but I believe i
### Match
The expliciy "match" type is going to look exactly at the type of a subsystem node, and return true (match) if it matches what the subsystem needs. For example, given this task:
The expliciy "match" type is going to look exactly at some exact value for a field in the metadata. It will return true (match) if it matches what the subsystem needs. For example, given this task:
```yaml
task:
Expand All @@ -133,10 +133,11 @@ task:
resources:
io:
match:
- type: shm
- field: type
value: shm
```
We would look for a node of type "shm" in the io subsystem that is directly attached (an edge) to a node in the dominant subsystem graph.
We would look for a node with field "type" and value "shm" in the io subsystem that is directly attached (an edge) to a node in the dominant subsystem graph.
### Range
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ resources:
type: core
task:
command:
- ior
- spack
slot: default
count:
per_slot: 1
Expand Down
3 changes: 2 additions & 1 deletion docs/examples/scheduler/jobspec-io.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,5 @@ task:
resources:
io:
match:
- type: shm
- field: type
value: shm
2 changes: 1 addition & 1 deletion docs/examples/scheduler/rainbow-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ scheduler:
name: match
cluster:
name: keebler
secret: 3994c1e7-9cc7-4b81-ab75-3b00128eda16
secret: 79643f8e-6408-4cd1-bd1a-76aa499d5864
graphdatabase:
name: memory
host: 127.0.0.1:50051
Expand Down
3 changes: 3 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ require (
github.com/akamensky/argparse v1.4.0
github.com/compspec/jobspec-go v0.0.0-20240319000127-8020a01a65da
github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe
github.com/fatih/color v1.16.0
github.com/google/uuid v1.6.0
github.com/mattn/go-sqlite3 v1.14.22
github.com/pkg/errors v0.9.1
Expand All @@ -17,6 +18,8 @@ require (

require (
github.com/golang/protobuf v1.5.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
Expand Down
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ github.com/compspec/jobspec-go v0.0.0-20240319000127-8020a01a65da h1:Uvfk4IgWMIi
github.com/compspec/jobspec-go v0.0.0-20240319000127-8020a01a65da/go.mod h1:BaJyxaOhESe2DD4lqBdwTEWOw0TaTZVJGPrFh6KyXQM=
github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe h1:Tk//RW3uKn4A7N8gpHRXs+ZGlR7Fxkwh+4/Iml0GBV4=
github.com/converged-computing/jsongraph-go v0.0.0-20240229082022-c6887a5a00fe/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
Expand All @@ -14,6 +16,11 @@ github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
Expand All @@ -22,6 +29,8 @@ github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 h1:lZUw3E0/J3roVtGQ+SCrUrg3ON6Ng
github.com/santhosh-tekuri/jsonschema/v5 v5.3.1/go.mod h1:uToXkOrWAZ6/Oc07xWQrPOhJotwFIyu2bBVN41fcDUY=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
Expand Down
178 changes: 178 additions & 0 deletions pkg/logger/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,178 @@
package logger

// TODO would like to use slog when we can use go 1.21!

import (
"fmt"
"log"
"os"

"github.com/fatih/color"
)

const (
LevelNone = iota
LevelInfo
LevelWarning
LevelError
LevelVerbose
LevelDebug
)

var (
DefaultLevel = 3
logger *RainbowLogger
)

type RainbowLogger struct {
level int
Filename string
handle *os.File
}

// Start opens a file handle, if it's desired to write to file
func (l *RainbowLogger) Start() (*log.Logger, error) {
f, err := os.OpenFile(l.Filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm)
if err != nil {
return nil, err
}
logger := log.New(f, "", 0)
l.handle = f
return logger, nil
}

// Stop closes the file handle, if defined
func (l *RainbowLogger) Stop() error {
if l.handle != nil {
return l.handle.Close()
}
return nil
}

// Logging functions with formatting
func Infof(message ...any) error {
return logger.logFormat(LevelInfo, message...)
}

func Errorf(message ...any) error {
color.Set(color.FgRed)
err := logger.logFormat(LevelError, message...)
color.Unset()
return err
}
func Debugf(message ...any) error {
color.Set(color.FgBlue)
err := logger.logFormat(LevelDebug, message...)
color.Unset()
return err

}
func Verbosef(message ...any) error {
color.Set(color.FgMagenta)
err := logger.logFormat(LevelVerbose, message...)
color.Unset()
return err

}
func Warningf(message ...any) error {
color.Set(color.FgYellow)
err := logger.logFormat(LevelWarning, message...)
color.Unset()
return err
}

// And without!
func Info(message string) error {
return logger.log(LevelInfo, message)
}
func Error(message string) error {
color.Set(color.FgRed)
err := logger.log(LevelError, message)
color.Unset()
return err

}
func Debug(message string) error {
color.Set(color.FgBlue)
err := logger.log(LevelDebug, message)
color.Unset()
return err
}
func Verbose(message string) error {
color.Set(color.FgMagenta)
err := logger.log(LevelVerbose, message)
color.Unset()
return err
}
func Warning(message string) error {
color.Set(color.FgYellow)
err := logger.log(LevelWarning, message)
color.Unset()
return err
}

// log prints (without formatting) to the log
func (l *RainbowLogger) log(level int, message string) error {
if l.Filename != "" {
l.logToFile(level, message)
}
if level >= l.level {
fmt.Println(message)
}
return nil
}

// logFormat is the shared class function for actually printing to the log
func (l *RainbowLogger) logFormat(level int, message ...any) error {
if l.Filename != "" {
l.logFormatToFile(level, message)
}
// Otherwise just print! Simple and dumb
prolog := message[0].(string)
rest := message[1:]
if level <= l.level {
fmt.Printf(prolog, rest...)
}
return nil
}

// logFormatToFile writes to file if the rainbow logger is set to do that
func (l *RainbowLogger) logFormatToFile(level int, message ...any) error {
logger, err := l.Start()
if err != nil {
return err
}
// Assume the prolog (to be formatted) is at index 0
prolog := message[0].(string)
rest := message[1:]
if level <= l.level {
logger.Printf(prolog, rest...)
}
return l.Stop()
}

// logToFile writes to file if the rainbow logger is set to do that
func (l *RainbowLogger) logToFile(level int, message string) error {
logger, err := l.Start()
if err != nil {
return err
}
if level <= l.level {
logger.Println(message)
}
return l.Stop()
}

// WriteToFile will set the global level and filename
func WriteToFile(filename string) {
logger.Filename = filename
}

// SetLevel sets the global logging level
func SetLevel(level int) {
logger.level = level
}

func init() {
logger = &RainbowLogger{level: LevelWarning}
}
Loading

0 comments on commit ef4376e

Please sign in to comment.