From ff216fdc5255173a37f123c07b939becef041ebd Mon Sep 17 00:00:00 2001 From: Quinton Pike Date: Thu, 21 Mar 2019 14:10:17 -0400 Subject: [PATCH] Had to use easyjson ( json encoding ) for values. Gob wouldn't allow unknown structures. --- errand-routes.go | 21 +- errand.go | 4 +- errands-routes.go | 34 +-- errands-server.go | 2 +- glide.lock | 8 +- glide.yaml | 1 + main_easyjson.go | 656 ++++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 684 insertions(+), 42 deletions(-) create mode 100644 main_easyjson.go diff --git a/errand-routes.go b/errand-routes.go index e457d51..5682f2d 100644 --- a/errand-routes.go +++ b/errand-routes.go @@ -4,10 +4,8 @@ package main import ( // "fmt" - "bytes" "errors" "net/http" - "encoding/gob" gin "github.com/gin-gonic/gin" badger "github.com/dgraph-io/badger" ) @@ -111,7 +109,7 @@ func ( s *ErrandsServer ) failedErrand( c *gin.Context ){ type CompletedRequest struct { - Results string `json:"results"` + Results *gin.H `json:"results"` } func ( s *ErrandsServer ) completeErrand( c *gin.Context ){ var updatedErrand *Errand @@ -268,21 +266,16 @@ func ( s *ErrandsServer ) UpdateErrandByID( id string, fn func( *Errand ) error item, err := txn.Get([]byte( id )); if err != nil { return err } - err = item.Value(func(val []byte) error { - var bytesBuffer bytes.Buffer - dec := gob.NewDecoder( &bytesBuffer ) - _, err := bytesBuffer.Write( val ); if err != nil { + err = item.Value(func(v []byte) error { + errand := &Errand{} + err := errand.UnmarshalJSON( v ); if err != nil { return err } - var errand Errand - err = dec.Decode( &errand ); if err != nil { + err = fn( errand ); if err != nil { return err } - err = fn( &errand ); if err != nil { - return err - } - updatedErrand = &errand - err = s.saveErrand( txn, &errand ); if err != nil { + updatedErrand = errand + err = s.saveErrand( txn, errand ); if err != nil { return err } return nil diff --git a/errand.go b/errand.go index 80ad92f..c7dd9f8 100644 --- a/errand.go +++ b/errand.go @@ -12,6 +12,7 @@ import ( var ErrandStatuses []string = []string{"inactive", "active", "failed", "completed"} +//easyjson:json type Errand struct { // General Attributes: @@ -27,7 +28,7 @@ type Errand struct { Data *gin.H `json:"data,omitempty"` Created int64 `json:"created"` Status string `json:"status,omitempty"` - Results string `json:"results,omitempty"` + Results *gin.H `json:"results,omitempty"` // Internal attributes: Progress float64 `json:"progress"` @@ -40,6 +41,7 @@ type Errand struct { var LogSeverities []string = []string{ "INFO", "WARNING", "ERROR" } +//easyjson:json type Log struct { Severity string `json:"severity" binding:"required"` Message string `json:"message" binding:"required"` diff --git a/errands-routes.go b/errands-routes.go index 84f278e..541bd1b 100644 --- a/errands-routes.go +++ b/errands-routes.go @@ -8,10 +8,8 @@ import ( "log" "sort" // "time" - "bytes" "errors" "net/http" - "encoding/gob" "encoding/json" gin "github.com/gin-gonic/gin" badger "github.com/dgraph-io/badger" @@ -103,12 +101,10 @@ func ( s *ErrandsServer ) saveErrand( txn *badger.Txn, errand *Errand ) error { if !contains(ErrandStatuses, errand.Status) { return errors.New("Invalid errand status state") } - var bytesBuffer bytes.Buffer - enc := gob.NewEncoder(&bytesBuffer) - err := enc.Encode( errand ); if err != nil { + bytes, err := errand.MarshalJSON(); if err != nil { return err } - return txn.Set([]byte(errand.ID), bytesBuffer.Bytes()) + return txn.Set([]byte(errand.ID), bytes) } @@ -237,15 +233,12 @@ func ( s *ErrandsServer ) processErrand( c *gin.Context ){ for it.Rewind(); it.Valid(); it.Next() { item := it.Item() err := item.Value(func( v []byte ) error { - var bytesBuffer bytes.Buffer - dec := gob.NewDecoder( &bytesBuffer ) - _, err := bytesBuffer.Write( v ); if err != nil { - return err - } - var errand Errand - err = dec.Decode( &errand ); if err != nil { + + errand := &Errand{} + err := errand.UnmarshalJSON( v ); if err != nil { return err } + if errand.Status != "inactive" { return nil } @@ -253,7 +246,7 @@ func ( s *ErrandsServer ) processErrand( c *gin.Context ){ return nil } // Add to list of errands we could possibly process: - errands = append( errands, &errand ) + errands = append( errands, errand ) return nil }) if err != nil { @@ -323,17 +316,12 @@ func ( s *ErrandsServer ) GetErrandsBy( fn func ( *Errand ) bool ) ( []*Errand, for it.Rewind(); it.Valid(); it.Next() { item := it.Item() err := item.Value(func( v []byte ) error { - var bytesBuffer bytes.Buffer - dec := gob.NewDecoder( &bytesBuffer ) - _, err := bytesBuffer.Write( v ); if err != nil { - return err - } - var errand Errand - err = dec.Decode( &errand ); if err != nil { + errand := &Errand{} + err := errand.UnmarshalJSON( v ); if err != nil { return err } - if( fn( &errand ) ){ - errands = append( errands, &errand ) + if( fn( errand ) ){ + errands = append( errands, errand ) } return nil }) diff --git a/errands-server.go b/errands-server.go index 4c5177a..2c2aad2 100644 --- a/errands-server.go +++ b/errands-server.go @@ -21,7 +21,7 @@ import ( - +//easyjson:json type Notification struct { Event string `json:"event"` Errand Errand `json:"errand,omitempty"` diff --git a/glide.lock b/glide.lock index c6936b9..f7323ec 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 726601405908e01023f1c1b7c499c6d95d06075079bd60beaf2934f00644913c -updated: 2019-03-21T00:49:10.246016-04:00 +hash: 611257d7204466869e78430a6fab8478dc479dcef4721262a9c16cb45e61710d +updated: 2019-03-21T13:54:02.935666-04:00 imports: - name: github.com/AndreasBriese/bbloom version: e2d15f34fcf99d5dbb871c820ec73f710fca9815 @@ -35,6 +35,8 @@ imports: version: 0ff49de124c6f76f8494e194af75bde0f1a49a29 - name: github.com/kelseyhightower/envconfig version: f611eb38b3875cc3bd991ca91c51d06446afa14c +- name: github.com/mailru/easyjson + version: 1de009706dbeb9d05f18586f0735fcdb7c524481 - name: github.com/mattn/go-isatty version: c2a7a6ca930a4cd0bc33a3f298eb71960732a3a7 - name: github.com/modern-go/concurrent @@ -53,7 +55,7 @@ imports: - internal/timeseries - trace - name: golang.org/x/sys - version: 6c81ef8f67ca3f42fc9cd71dfbd5f35b0c4b5771 + version: f7bb7a8bee54210937e93ec56d007d892c1f0580 subpackages: - unix - name: gopkg.in/go-playground/validator.v8 diff --git a/glide.yaml b/glide.yaml index 13da9bb..9858bdc 100644 --- a/glide.yaml +++ b/glide.yaml @@ -14,3 +14,4 @@ import: version: ^0.0.1 - package: github.com/kelseyhightower/envconfig version: ^1.3.0 +- package: github.com/mailru/easyjson diff --git a/main_easyjson.go b/main_easyjson.go new file mode 100644 index 0000000..51a5f6a --- /dev/null +++ b/main_easyjson.go @@ -0,0 +1,656 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package main + +import ( + json "encoding/json" + gin "github.com/gin-gonic/gin" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjson89aae3efDecodeGithubComPolygonIoErrandsServer(in *jlexer.Lexer, out *Notification) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "event": + out.Event = string(in.String()) + case "errand": + (out.Errand).UnmarshalEasyJSON(in) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson89aae3efEncodeGithubComPolygonIoErrandsServer(out *jwriter.Writer, in Notification) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"event\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Event)) + } + if true { + const prefix string = ",\"errand\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + (in.Errand).MarshalEasyJSON(out) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v Notification) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson89aae3efEncodeGithubComPolygonIoErrandsServer(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v Notification) MarshalEasyJSON(w *jwriter.Writer) { + easyjson89aae3efEncodeGithubComPolygonIoErrandsServer(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *Notification) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson89aae3efDecodeGithubComPolygonIoErrandsServer(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *Notification) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson89aae3efDecodeGithubComPolygonIoErrandsServer(l, v) +} +func easyjson89aae3efDecodeGithubComPolygonIoErrandsServer1(in *jlexer.Lexer, out *Log) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "severity": + out.Severity = string(in.String()) + case "message": + out.Message = string(in.String()) + case "timestamp": + out.Timestamp = int64(in.Int64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson89aae3efEncodeGithubComPolygonIoErrandsServer1(out *jwriter.Writer, in Log) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"severity\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Severity)) + } + { + const prefix string = ",\"message\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Message)) + } + { + const prefix string = ",\"timestamp\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int64(int64(in.Timestamp)) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v Log) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson89aae3efEncodeGithubComPolygonIoErrandsServer1(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v Log) MarshalEasyJSON(w *jwriter.Writer) { + easyjson89aae3efEncodeGithubComPolygonIoErrandsServer1(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *Log) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson89aae3efDecodeGithubComPolygonIoErrandsServer1(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *Log) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson89aae3efDecodeGithubComPolygonIoErrandsServer1(l, v) +} +func easyjson89aae3efDecodeGithubComPolygonIoErrandsServer2(in *jlexer.Lexer, out *Errand) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = string(in.String()) + case "name": + out.Name = string(in.String()) + case "type": + out.Type = string(in.String()) + case "options": + easyjson89aae3efDecode(in, &out.Options) + case "data": + if in.IsNull() { + in.Skip() + out.Data = nil + } else { + if out.Data == nil { + out.Data = new(gin.H) + } + if in.IsNull() { + in.Skip() + } else { + in.Delim('{') + if !in.IsDelim('}') { + *out.Data = make(gin.H) + } else { + *out.Data = nil + } + for !in.IsDelim('}') { + key := string(in.String()) + in.WantColon() + var v1 interface{} + if m, ok := v1.(easyjson.Unmarshaler); ok { + m.UnmarshalEasyJSON(in) + } else if m, ok := v1.(json.Unmarshaler); ok { + _ = m.UnmarshalJSON(in.Raw()) + } else { + v1 = in.Interface() + } + (*out.Data)[key] = v1 + in.WantComma() + } + in.Delim('}') + } + } + case "created": + out.Created = int64(in.Int64()) + case "status": + out.Status = string(in.String()) + case "results": + if in.IsNull() { + in.Skip() + out.Results = nil + } else { + if out.Results == nil { + out.Results = new(gin.H) + } + if in.IsNull() { + in.Skip() + } else { + in.Delim('{') + if !in.IsDelim('}') { + *out.Results = make(gin.H) + } else { + *out.Results = nil + } + for !in.IsDelim('}') { + key := string(in.String()) + in.WantColon() + var v2 interface{} + if m, ok := v2.(easyjson.Unmarshaler); ok { + m.UnmarshalEasyJSON(in) + } else if m, ok := v2.(json.Unmarshaler); ok { + _ = m.UnmarshalJSON(in.Raw()) + } else { + v2 = in.Interface() + } + (*out.Results)[key] = v2 + in.WantComma() + } + in.Delim('}') + } + } + case "progress": + out.Progress = float64(in.Float64()) + case "attempts": + out.Attempts = int(in.Int()) + case "started": + out.Started = int64(in.Int64()) + case "failed": + out.Failed = int64(in.Int64()) + case "compelted": + out.Completed = int64(in.Int64()) + case "logs": + if in.IsNull() { + in.Skip() + out.Logs = nil + } else { + in.Delim('[') + if out.Logs == nil { + if !in.IsDelim(']') { + out.Logs = make([]Log, 0, 1) + } else { + out.Logs = []Log{} + } + } else { + out.Logs = (out.Logs)[:0] + } + for !in.IsDelim(']') { + var v3 Log + (v3).UnmarshalEasyJSON(in) + out.Logs = append(out.Logs, v3) + in.WantComma() + } + in.Delim(']') + } + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson89aae3efEncodeGithubComPolygonIoErrandsServer2(out *jwriter.Writer, in Errand) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"id\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.ID)) + } + { + const prefix string = ",\"name\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Name)) + } + { + const prefix string = ",\"type\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Type)) + } + { + const prefix string = ",\"options\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + easyjson89aae3efEncode(out, in.Options) + } + if in.Data != nil { + const prefix string = ",\"data\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + if *in.Data == nil && (out.Flags&jwriter.NilMapAsEmpty) == 0 { + out.RawString(`null`) + } else { + out.RawByte('{') + v4First := true + for v4Name, v4Value := range *in.Data { + if v4First { + v4First = false + } else { + out.RawByte(',') + } + out.String(string(v4Name)) + out.RawByte(':') + if m, ok := v4Value.(easyjson.Marshaler); ok { + m.MarshalEasyJSON(out) + } else if m, ok := v4Value.(json.Marshaler); ok { + out.Raw(m.MarshalJSON()) + } else { + out.Raw(json.Marshal(v4Value)) + } + } + out.RawByte('}') + } + } + { + const prefix string = ",\"created\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int64(int64(in.Created)) + } + if in.Status != "" { + const prefix string = ",\"status\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.String(string(in.Status)) + } + if in.Results != nil { + const prefix string = ",\"results\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + if *in.Results == nil && (out.Flags&jwriter.NilMapAsEmpty) == 0 { + out.RawString(`null`) + } else { + out.RawByte('{') + v5First := true + for v5Name, v5Value := range *in.Results { + if v5First { + v5First = false + } else { + out.RawByte(',') + } + out.String(string(v5Name)) + out.RawByte(':') + if m, ok := v5Value.(easyjson.Marshaler); ok { + m.MarshalEasyJSON(out) + } else if m, ok := v5Value.(json.Marshaler); ok { + out.Raw(m.MarshalJSON()) + } else { + out.Raw(json.Marshal(v5Value)) + } + } + out.RawByte('}') + } + } + { + const prefix string = ",\"progress\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Float64(float64(in.Progress)) + } + { + const prefix string = ",\"attempts\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int(int(in.Attempts)) + } + if in.Started != 0 { + const prefix string = ",\"started\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int64(int64(in.Started)) + } + if in.Failed != 0 { + const prefix string = ",\"failed\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int64(int64(in.Failed)) + } + if in.Completed != 0 { + const prefix string = ",\"compelted\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int64(int64(in.Completed)) + } + if len(in.Logs) != 0 { + const prefix string = ",\"logs\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + { + out.RawByte('[') + for v6, v7 := range in.Logs { + if v6 > 0 { + out.RawByte(',') + } + (v7).MarshalEasyJSON(out) + } + out.RawByte(']') + } + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v Errand) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson89aae3efEncodeGithubComPolygonIoErrandsServer2(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v Errand) MarshalEasyJSON(w *jwriter.Writer) { + easyjson89aae3efEncodeGithubComPolygonIoErrandsServer2(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *Errand) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson89aae3efDecodeGithubComPolygonIoErrandsServer2(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *Errand) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson89aae3efDecodeGithubComPolygonIoErrandsServer2(l, v) +} +func easyjson89aae3efDecode(in *jlexer.Lexer, out *struct { + TTL int `json:"ttl,omitempty"` + Retries int `json:"retries,omitempty"` + Priority int `json:"priority,omitempty"` + DeleteOnCompleted bool `json:"deleteOnCompleted,omitempty"` +}) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeString() + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "ttl": + out.TTL = int(in.Int()) + case "retries": + out.Retries = int(in.Int()) + case "priority": + out.Priority = int(in.Int()) + case "deleteOnCompleted": + out.DeleteOnCompleted = bool(in.Bool()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson89aae3efEncode(out *jwriter.Writer, in struct { + TTL int `json:"ttl,omitempty"` + Retries int `json:"retries,omitempty"` + Priority int `json:"priority,omitempty"` + DeleteOnCompleted bool `json:"deleteOnCompleted,omitempty"` +}) { + out.RawByte('{') + first := true + _ = first + if in.TTL != 0 { + const prefix string = ",\"ttl\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int(int(in.TTL)) + } + if in.Retries != 0 { + const prefix string = ",\"retries\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int(int(in.Retries)) + } + if in.Priority != 0 { + const prefix string = ",\"priority\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Int(int(in.Priority)) + } + if in.DeleteOnCompleted { + const prefix string = ",\"deleteOnCompleted\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Bool(bool(in.DeleteOnCompleted)) + } + out.RawByte('}') +}