diff --git a/src/ts-publishing-api-go/go.mod b/src/ts-publishing-api-go/go.mod index ee475fd..c76231c 100644 --- a/src/ts-publishing-api-go/go.mod +++ b/src/ts-publishing-api-go/go.mod @@ -1,9 +1,12 @@ module github.com/turbosquid/ts-publishing-api-go -go 1.12 +go 1.21 require ( + github.com/avast/retry-go/v4 v4.5.0 github.com/aws/aws-sdk-go v1.34.18 github.com/google/jsonapi v0.0.0-20200825183604-3e3da1210d0c gopkg.in/yaml.v2 v2.3.0 ) + +require github.com/jmespath/go-jmespath v0.3.0 // indirect diff --git a/src/ts-publishing-api-go/go.sum b/src/ts-publishing-api-go/go.sum index 397ff40..8c93283 100644 --- a/src/ts-publishing-api-go/go.sum +++ b/src/ts-publishing-api-go/go.sum @@ -1,20 +1,32 @@ +github.com/avast/retry-go/v4 v4.5.0 h1:QoRAZZ90cj5oni2Lsgl2GW8mNTnUCnmpx/iKpwVisHg= +github.com/avast/retry-go/v4 v4.5.0/go.mod h1:7hLEXp0oku2Nir2xBAsg0PTphp9z71bN5Aq1fboC3+I= github.com/aws/aws-sdk-go v1.34.18 h1:Mo/Clq3u1dQFzpg8YQqBii8m+Vl3fWIfHi6kXs5wpuM= github.com/aws/aws-sdk-go v1.34.18/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/google/jsonapi v0.0.0-20200825183604-3e3da1210d0c h1:p2sf7ppyG09K/Er1FeLSDeSjaEvsycQSdTB/il3JhCc= github.com/google/jsonapi v0.0.0-20200825183604-3e3da1210d0c/go.mod h1:XSx4m2SziAqk9DXY9nz659easTq4q6TyrpYd9tHSm0g= github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/src/ts-publishing-api-go/upload.go b/src/ts-publishing-api-go/upload.go index 07f1c20..4d472a9 100644 --- a/src/ts-publishing-api-go/upload.go +++ b/src/ts-publishing-api-go/upload.go @@ -2,13 +2,16 @@ package main import ( "bytes" + "errors" "fmt" "log" "net/http" "os" "path/filepath" + "strings" "time" + retry "github.com/avast/retry-go/v4" "github.com/aws/aws-sdk-go/aws" awscreds "github.com/aws/aws-sdk-go/aws/credentials" awssession "github.com/aws/aws-sdk-go/aws/session" @@ -37,44 +40,66 @@ type Upload struct { } func (credentials *Credentials) Upload(directory string, filePath string, settings Settings) (error, int) { - if err := credentials.checkExpired(settings); err != nil { - log.Fatalf("Failure getting credentials: %s", err) - } - log.Printf("Uploading file %s", filePath) + fileId, err := retry.DoWithData( + func() (int, error) { + if err := credentials.checkExpired(settings); err != nil { + log.Fatalf("Failure getting credentials: %s", err) + } + + err, upload := credentials.UploadFile(fmt.Sprintf("%s/%s", directory, filePath)) + if err != nil { + log.Fatalf("Failure uploading file: %s", err) + } + + if settings.Debug { + log.Printf("Processing file %s", filePath) + } + if err = processUpload(settings, &upload); err != nil { + log.Fatalf("Failure processing upload: %s", err) + } + + if settings.Debug { + log.Printf("Polling process file %s: %s", filePath, upload.Id) + } + start := time.Now() + sleep := 1 + for { + upload.Poll(settings) + t := time.Now() + if (upload.Status != "queued" && upload.Status != "processing") || int(t.Sub(start).Seconds()) > settings.UploadTimeout { + break + } else { + time.Sleep(time.Duration(sleep)) + sleep = sleep + 1 + } + } + + if upload.Status != "success" { + return 0, errors.New(upload.Message) + } + + log.Printf("Upload %s complete", upload.FileId) + + return upload.FileId, err + }, + + retry.Attempts(3), + + retry.RetryIf(func(err error) bool { + if strings.Contains(err.Error(), "virus check failure") { + log.Printf("Check failure for %s, retrying...", filePath) + return true + } else { + return false + } + }), + ) - err, upload := credentials.UploadFile(fmt.Sprintf("%s/%s", directory, filePath)) if err != nil { - log.Fatalf("Failure uploading file: %s", err) - } - - if settings.Debug { - log.Printf("Processing file %s", filePath) - } - if err = processUpload(settings, &upload); err != nil { - log.Fatalf("Failure processing upload: %s", err) - } - - if settings.Debug { - log.Printf("Polling process file %s: %s", filePath, upload.Id) - } - start := time.Now() - sleep := 1 - for { - upload.Poll(settings) - t := time.Now() - if (upload.Status != "queued" && upload.Status != "processing") || int(t.Sub(start).Seconds()) > settings.UploadTimeout { - break - } else { - time.Sleep(time.Duration(sleep)) - sleep = sleep + 1 - } - } - - if upload.Status != "success" { - log.Fatalf("Upload process failed for %s: %s", filePath, upload.Status) + log.Fatalf("Upload process failed for %s: %s", filePath, err) } - return err, upload.FileId + return err, fileId } func (credentials *Credentials) UploadFile(source string) (error, Upload) {