From 98a7571efeebaeee19dd1252020f98472c61ee7d Mon Sep 17 00:00:00 2001 From: a Date: Tue, 5 Dec 2023 12:29:58 +0000 Subject: [PATCH 01/27] add antiraid-specific get gateway patch --- lib/discord.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index cf99f86..1f44e2f 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -5,7 +5,6 @@ import ( "crypto/tls" "encoding/json" "errors" - "github.com/sirupsen/logrus" "io" "io/ioutil" "math" @@ -14,6 +13,8 @@ import ( "strconv" "strings" "time" + + "github.com/sirupsen/logrus" ) var client *http.Client @@ -242,6 +243,28 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC RequestHistogram.With(map[string]string{"route": route, "status": status, "method": method, "clientId": identifier.(string)}).Observe(elapsed) } + + // ANTIRAID-SPECIFIC, Patch /gateway/bot to return a link to gateway-proxy + if path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { + var data map[string]any + + err := json.NewDecoder(discordResp.Body).Decode(&data) + + if err != nil { + return nil, err + } + + data["url"] = "ws://0.0.0.0:7878" + + bytes, err := json.Marshal(data) + + if err != nil { + return nil, err + } + + discordResp.Body = ioutil.NopCloser(strings.NewReader(string(bytes))) + } + return discordResp, err } From 3d26a934720359499828dca9bc45242bcadc72b7 Mon Sep 17 00:00:00 2001 From: a Date: Wed, 6 Dec 2023 04:10:52 +0000 Subject: [PATCH 02/27] proxy more --- lib/discord.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index 1f44e2f..7a245ef 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -245,7 +245,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC } // ANTIRAID-SPECIFIC, Patch /gateway/bot to return a link to gateway-proxy - if path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { + if path == "/api/gateway" || path == "/api/v9/gateway" || path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { var data map[string]any err := json.NewDecoder(discordResp.Body).Decode(&data) From 78d9a35f0d857f4c41a3b46487551c516d48540e Mon Sep 17 00:00:00 2001 From: a Date: Thu, 14 Dec 2023 16:36:56 +0000 Subject: [PATCH 03/27] caching --- lib/cache.go | 57 +++++++++++++++++++++++++++++++++++++ lib/discord.go | 76 ++++++++++++++++++++++++++++++++++++++++++++------ lib/queue.go | 9 +++--- 3 files changed, 129 insertions(+), 13 deletions(-) create mode 100644 lib/cache.go diff --git a/lib/cache.go b/lib/cache.go new file mode 100644 index 0000000..60997e1 --- /dev/null +++ b/lib/cache.go @@ -0,0 +1,57 @@ +package lib + +import ( + "net/http" + "time" +) + +type CacheEntry struct { + Data []byte + CreatedAt time.Time + ExpiresIn *time.Duration + Headers http.Header +} + +func (c *CacheEntry) Expired() bool { + if c.ExpiresIn == nil { + return false + } + return time.Since(c.CreatedAt) > *c.ExpiresIn +} + +type Cache struct { + entries map[string]*CacheEntry +} + +func NewCache() *Cache { + return &Cache{ + entries: make(map[string]*CacheEntry), + } +} + +func (c *Cache) Get(key string) *CacheEntry { + entry, ok := c.entries[key] + + if !ok { + return nil + } + + if entry.Expired() { + c.Delete(key) + return nil + } + + return entry +} + +func (c *Cache) Set(key string, entry *CacheEntry) { + c.entries[key] = entry +} + +func (c *Cache) Delete(key string) { + delete(c.entries, key) +} + +func (c *Cache) Clear() { + c.entries = make(map[string]*CacheEntry) +} diff --git a/lib/discord.go b/lib/discord.go index 7a245ef..7d4959a 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -1,6 +1,7 @@ package lib import ( + "bytes" "context" "crypto/tls" "encoding/json" @@ -23,8 +24,22 @@ var contextTimeout time.Duration var globalOverrideMap = make(map[string]uint) +var endpointCache = make(map[string]*Cache) + var disableRestLimitDetection = false +var cacheEndpoints = map[string]time.Duration{ + "/api/users/@me": 10 * time.Minute, + "/api/v9/users/@me": 10 * time.Minute, + "/api/v10/users/@me": 10 * time.Minute, + "/api/gateway": 60 * time.Minute, + "/api/v9/gateway": 60 * time.Minute, + "/api/v10/gateway": 60 * time.Minute, + "/api/gateway/bot": 30 * time.Minute, + "/api/v9/gateway/bot": 30 * time.Minute, + "/api/v10/gateway/bot": 30 * time.Minute, +} + type BotGatewayResponse struct { SessionStartLimit map[string]int `json:"session_start_limit"` } @@ -201,7 +216,7 @@ func GetBotUser(token string) (*BotUserResponse, error) { return nil, errors.New("500 on users/@me") } - body, _ := ioutil.ReadAll(bot.Body) + body, _ := io.ReadAll(bot.Body) var s BotUserResponse @@ -214,6 +229,41 @@ func GetBotUser(token string) (*BotUserResponse, error) { } func doDiscordReq(ctx context.Context, path string, method string, body io.ReadCloser, header http.Header, query string) (*http.Response, error) { + identifier := ctx.Value("identifier") + if identifier == nil { + // Queues always have an identifier, if there's none in the context, we called the method from outside a queue + identifier = "Internal" + } + + identifierStr, ok := identifier.(string) + + if ok { + // Check endpoint cache + if endpointCache[identifierStr] != nil { + cacheEntry := endpointCache[identifierStr].Get(path) + + if cacheEntry != nil { + // Send cached response + logger.WithFields(logrus.Fields{ + "method": method, + "path": path, + "status": "200 (cached)", + }).Debug("Discord request") + + headers := cacheEntry.Headers.Clone() + headers.Set("X-Cached", "true") + + return &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewBuffer(cacheEntry.Data)), + Header: headers, + }, nil + } + } else { + endpointCache[identifierStr] = NewCache() + } + } + discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) if err != nil { return nil, err @@ -223,12 +273,6 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC startTime := time.Now() discordResp, err := client.Do(discordReq) - identifier := ctx.Value("identifier") - if identifier == nil { - // Queues always have an identifier, if there's none in the context, we called the method from outside a queue - identifier = "Internal" - } - if err == nil { route := GetMetricsPath(path) status := discordResp.Status @@ -244,7 +288,6 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC RequestHistogram.With(map[string]string{"route": route, "status": status, "method": method, "clientId": identifier.(string)}).Observe(elapsed) } - // ANTIRAID-SPECIFIC, Patch /gateway/bot to return a link to gateway-proxy if path == "/api/gateway" || path == "/api/v9/gateway" || path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { var data map[string]any @@ -262,7 +305,22 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC return nil, err } - discordResp.Body = ioutil.NopCloser(strings.NewReader(string(bytes))) + discordResp.Body = io.NopCloser(strings.NewReader(string(bytes))) + } + + if expiry, ok := cacheEndpoints[path]; ok { + if discordResp.StatusCode == 200 { + body, _ := io.ReadAll(discordResp.Body) + endpointCache[identifierStr].Set(path, &CacheEntry{ + Data: body, + CreatedAt: time.Now(), + ExpiresIn: &expiry, + Headers: discordResp.Header, + }) + + // Put body back into response + discordResp.Body = io.NopCloser(bytes.NewBuffer(body)) + } } return discordResp, err diff --git a/lib/queue.go b/lib/queue.go index 203383a..f20123f 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -3,15 +3,16 @@ package lib import ( "context" "errors" - "github.com/Clever/leakybucket" - "github.com/Clever/leakybucket/memory" - "github.com/sirupsen/logrus" "net/http" "strconv" "strings" "sync" "sync/atomic" "time" + + "github.com/Clever/leakybucket" + "github.com/Clever/leakybucket/memory" + "github.com/sirupsen/logrus" ) type QueueItem struct { @@ -82,7 +83,7 @@ func NewRequestQueue(processor func(ctx context.Context, item *QueueItem) (*http identifier := "NoAuth" if user != nil { queueType = Bot - identifier = user.Username + "#" + user.Discrim + identifier = user.Id } if queueType == Bearer { From ea5bc3d29c9039d398e8b7612f8370a568a1a34c Mon Sep 17 00:00:00 2001 From: a Date: Sun, 4 Feb 2024 05:45:53 +0000 Subject: [PATCH 04/27] fix --- lib/discord.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index 7d4959a..0317ad5 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -1,6 +1,7 @@ package lib import ( + "os" "bytes" "context" "crypto/tls" @@ -252,6 +253,10 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC headers := cacheEntry.Headers.Clone() headers.Set("X-Cached", "true") + // Set cache headers so bot won't be perpetually stuck + headers.Set("X-RateLimit-Limit", "5") + headers.Set("X-RateLimit-Remaining", "5") + headers.Set("X-RateLimit-Bucket", "cache") return &http.Response{ StatusCode: 200, @@ -288,6 +293,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC RequestHistogram.With(map[string]string{"route": route, "status": status, "method": method, "clientId": identifier.(string)}).Observe(elapsed) } + if os.Getenv("EXP") == "true" { if path == "/api/gateway" || path == "/api/v9/gateway" || path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { var data map[string]any @@ -297,7 +303,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC return nil, err } - data["url"] = "ws://0.0.0.0:7878" + data["url"] = "ws://0.0.0.0:3220" bytes, err := json.Marshal(data) @@ -307,6 +313,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC discordResp.Body = io.NopCloser(strings.NewReader(string(bytes))) } +} if expiry, ok := cacheEndpoints[path]; ok { if discordResp.StatusCode == 200 { From 9274dbe8b7b8d8a800a2922f3b6c1871cce757b9 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 11 Feb 2024 17:17:59 +0000 Subject: [PATCH 05/27] fix --- Makefile | 2 ++ gitignore | 1 + 2 files changed, 3 insertions(+) create mode 100644 Makefile create mode 100644 gitignore diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..2d79bcd --- /dev/null +++ b/Makefile @@ -0,0 +1,2 @@ +all: + CGO_ENABLED=0 go build -v diff --git a/gitignore b/gitignore new file mode 100644 index 0000000..4bddeaf --- /dev/null +++ b/gitignore @@ -0,0 +1 @@ +nirn-proxy From 73623aa003f3e1e3b1f371d0f1f88eae0f90fb76 Mon Sep 17 00:00:00 2001 From: a Date: Mon, 12 Feb 2024 12:13:22 +0000 Subject: [PATCH 06/27] add wsProxy support better --- lib/discord.go | 64 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 43 insertions(+), 21 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index 0317ad5..a191440 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -1,22 +1,21 @@ package lib import ( - "os" "bytes" "context" "crypto/tls" "encoding/json" "errors" + "github.com/sirupsen/logrus" "io" "io/ioutil" "math" "net" "net/http" + "os" "strconv" "strings" "time" - - "github.com/sirupsen/logrus" ) var client *http.Client @@ -41,6 +40,29 @@ var cacheEndpoints = map[string]time.Duration{ "/api/v10/gateway/bot": 30 * time.Minute, } +var wsProxy string + +func init() { + if len(os.Args) > 1 { + for _, arg := range os.Args[1:] { + argSplit := strings.SplitN(arg, "=", 2) + + if len(argSplit) < 2 { + continue + } + + switch argSplit[0] { + case "ws-proxy": + wsProxy = argSplit[1] + } + } + } + + if os.Getenv("WS_PROXY") != "" { + wsProxy = os.Getenv("WS_PROXY") + } +} + type BotGatewayResponse struct { SessionStartLimit map[string]int `json:"session_start_limit"` } @@ -253,10 +275,10 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC headers := cacheEntry.Headers.Clone() headers.Set("X-Cached", "true") - // Set cache headers so bot won't be perpetually stuck - headers.Set("X-RateLimit-Limit", "5") - headers.Set("X-RateLimit-Remaining", "5") - headers.Set("X-RateLimit-Bucket", "cache") + // Set cache headers so bot won't be perpetually stuck + headers.Set("X-RateLimit-Limit", "5") + headers.Set("X-RateLimit-Remaining", "5") + headers.Set("X-RateLimit-Bucket", "cache") return &http.Response{ StatusCode: 200, @@ -293,27 +315,27 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC RequestHistogram.With(map[string]string{"route": route, "status": status, "method": method, "clientId": identifier.(string)}).Observe(elapsed) } - if os.Getenv("EXP") == "true" { - if path == "/api/gateway" || path == "/api/v9/gateway" || path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { - var data map[string]any + if wsProxy != "" { + if path == "/api/gateway" || path == "/api/v9/gateway" || path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { + var data map[string]any - err := json.NewDecoder(discordResp.Body).Decode(&data) + err := json.NewDecoder(discordResp.Body).Decode(&data) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - data["url"] = "ws://0.0.0.0:3220" + data["url"] = wsProxy - bytes, err := json.Marshal(data) + bytes, err := json.Marshal(data) - if err != nil { - return nil, err - } + if err != nil { + return nil, err + } - discordResp.Body = io.NopCloser(strings.NewReader(string(bytes))) + discordResp.Body = io.NopCloser(strings.NewReader(string(bytes))) + } } -} if expiry, ok := cacheEndpoints[path]; ok { if discordResp.StatusCode == 200 { From 003174aea99ee2e0ad4b0ade92b9351103e5cb0f Mon Sep 17 00:00:00 2001 From: Antiraid Date: Mon, 19 Feb 2024 02:08:41 +0000 Subject: [PATCH 07/27] . --- lib/discord.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/discord.go b/lib/discord.go index a191440..2f1a3f0 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -54,6 +54,10 @@ func init() { switch argSplit[0] { case "ws-proxy": wsProxy = argSplit[1] + break + case "port": + os.Setenv("PORT", argSplit[1]) + break } } } From af34aac570109638bee8941857900a754ae3e808 Mon Sep 17 00:00:00 2001 From: a Date: Sun, 25 Feb 2024 04:17:16 +0000 Subject: [PATCH 08/27] d --- lib/discord.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/discord.go b/lib/discord.go index 2f1a3f0..9de1634 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -38,6 +38,8 @@ var cacheEndpoints = map[string]time.Duration{ "/api/gateway/bot": 30 * time.Minute, "/api/v9/gateway/bot": 30 * time.Minute, "/api/v10/gateway/bot": 30 * time.Minute, + "/api/v9/applications/@me": 5 * time.Minute, + "/api/v10/applications/@me": 5 * time.Minute, } var wsProxy string @@ -262,6 +264,8 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC identifier = "Internal" } + logger.Info(method, path+"?"+query) + identifierStr, ok := identifier.(string) if ok { From 3bc2e30c9fdc405148a47c28e27926c8b570b7b9 Mon Sep 17 00:00:00 2001 From: Sanandan Sashikumar Date: Thu, 2 May 2024 16:10:16 +0000 Subject: [PATCH 09/27] add support for ratelimit-over-408 --- lib/discord.go | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index 9de1634..c0e7285 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -6,9 +6,7 @@ import ( "crypto/tls" "encoding/json" "errors" - "github.com/sirupsen/logrus" "io" - "io/ioutil" "math" "net" "net/http" @@ -16,6 +14,8 @@ import ( "strconv" "strings" "time" + + "github.com/sirupsen/logrus" ) var client *http.Client @@ -29,20 +29,21 @@ var endpointCache = make(map[string]*Cache) var disableRestLimitDetection = false var cacheEndpoints = map[string]time.Duration{ - "/api/users/@me": 10 * time.Minute, - "/api/v9/users/@me": 10 * time.Minute, - "/api/v10/users/@me": 10 * time.Minute, - "/api/gateway": 60 * time.Minute, - "/api/v9/gateway": 60 * time.Minute, - "/api/v10/gateway": 60 * time.Minute, - "/api/gateway/bot": 30 * time.Minute, - "/api/v9/gateway/bot": 30 * time.Minute, - "/api/v10/gateway/bot": 30 * time.Minute, - "/api/v9/applications/@me": 5 * time.Minute, + "/api/users/@me": 10 * time.Minute, + "/api/v9/users/@me": 10 * time.Minute, + "/api/v10/users/@me": 10 * time.Minute, + "/api/gateway": 60 * time.Minute, + "/api/v9/gateway": 60 * time.Minute, + "/api/v10/gateway": 60 * time.Minute, + "/api/gateway/bot": 30 * time.Minute, + "/api/v9/gateway/bot": 30 * time.Minute, + "/api/v10/gateway/bot": 30 * time.Minute, + "/api/v9/applications/@me": 5 * time.Minute, "/api/v10/applications/@me": 5 * time.Minute, } var wsProxy string +var ratelimitOver408 bool func init() { if len(os.Args) > 1 { @@ -56,10 +57,12 @@ func init() { switch argSplit[0] { case "ws-proxy": wsProxy = argSplit[1] - break case "port": os.Setenv("PORT", argSplit[1]) - break + case "ratelimit-over-408": + ratelimitOver408 = true + default: + logrus.Fatal("Unknown argument: ", argSplit[0]) } } } @@ -67,6 +70,10 @@ func init() { if os.Getenv("WS_PROXY") != "" { wsProxy = os.Getenv("WS_PROXY") } + + if os.Getenv("RATELIMIT_OVER_408") != "" { + ratelimitOver408 = os.Getenv("RATELIMIT_OVER_408") == "true" + } } type BotGatewayResponse struct { @@ -206,7 +213,7 @@ func GetBotGlobalLimit(token string, user *BotUserResponse) (uint, error) { return 0, errors.New("500 on gateway/bot") } - body, _ := ioutil.ReadAll(bot.Body) + body, _ := io.ReadAll(bot.Body) var s BotGatewayResponse @@ -373,7 +380,12 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error if err != nil { if ctx.Err() == context.DeadlineExceeded { - res.WriteHeader(408) + if ratelimitOver408 { + res.WriteHeader(429) + res.Header().Add("Reset-After", "2") + } else { + res.WriteHeader(408) + } } else { res.WriteHeader(500) } From 51e6ac5716e0ad8cd38c330959d9670a1a286409 Mon Sep 17 00:00:00 2001 From: Sanandan Sashikumar Date: Thu, 2 May 2024 16:14:29 +0000 Subject: [PATCH 10/27] fix headers --- lib/discord.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index c0e7285..990a1a7 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -290,7 +290,8 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC headers := cacheEntry.Headers.Clone() headers.Set("X-Cached", "true") - // Set cache headers so bot won't be perpetually stuck + + // Set rl headers so bot won't be perpetually stuck headers.Set("X-RateLimit-Limit", "5") headers.Set("X-RateLimit-Remaining", "5") headers.Set("X-RateLimit-Bucket", "cache") @@ -382,7 +383,20 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error if ctx.Err() == context.DeadlineExceeded { if ratelimitOver408 { res.WriteHeader(429) - res.Header().Add("Reset-After", "2") + res.Header().Add("Reset-After", "3") + + // Set rl headers so bot won't be perpetually stuck + if res.Header().Get("X-RateLimit-Limit") == "" { + res.Header().Set("X-RateLimit-Limit", "5") + } + if res.Header().Get("X-RateLimit-Remaining") == "" { + res.Header().Set("X-RateLimit-Remaining", "0") + } + + if res.Header().Get("X-RateLimit-Bucket") == "" { + res.Header().Set("X-RateLimit-Bucket", "proxyTimeout") + } + } else { res.WriteHeader(408) } From a13a8932da2e3e6514a9cfe1187fb6742d4021ad Mon Sep 17 00:00:00 2001 From: Sanandan Sashikumar Date: Thu, 2 May 2024 16:15:48 +0000 Subject: [PATCH 11/27] set scope on 408 rls to shared --- lib/discord.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/discord.go b/lib/discord.go index 990a1a7..af459f1 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -397,6 +397,11 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error res.Header().Set("X-RateLimit-Bucket", "proxyTimeout") } + // Default to 'shared' so the bot doesn't think its + // against them + if res.Header().Get("X-RateLimit-Scope") == "" { + res.Header().Set("X-RateLimit-Scope", "shared") + } } else { res.WriteHeader(408) } From f6785b5d7dfa1a69eb70dd60b40dc7caa1c894b7 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Fri, 3 May 2024 11:16:20 +0530 Subject: [PATCH 12/27] Update discord.go --- lib/discord.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index af459f1..7b66f50 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -51,7 +51,7 @@ func init() { argSplit := strings.SplitN(arg, "=", 2) if len(argSplit) < 2 { - continue + argSplit = append(argSplit, "") } switch argSplit[0] { From a0b231dc88a0239853e4f8df90d8163746af745e Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 18:43:44 +0530 Subject: [PATCH 13/27] Update lib/discord.go Apply suffix change Co-authored-by: Germano Eichenberg --- lib/discord.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index 7b66f50..dae55c3 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -332,7 +332,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC } if wsProxy != "" { - if path == "/api/gateway" || path == "/api/v9/gateway" || path == "/api/gateway/bot" || path == "/api/v10/gateway/bot" { + if strings.HasSuffix(path, "/gateway") || strings.HasSuffix(path, "/gateway/bot") { var data map[string]any err := json.NewDecoder(discordResp.Body).Decode(&data) From 0549e9937ac556de07c881811beaf06422a2c26b Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 18:45:04 +0530 Subject: [PATCH 14/27] Revert queue change --- lib/queue.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/queue.go b/lib/queue.go index f20123f..df0be2c 100644 --- a/lib/queue.go +++ b/lib/queue.go @@ -83,7 +83,7 @@ func NewRequestQueue(processor func(ctx context.Context, item *QueueItem) (*http identifier := "NoAuth" if user != nil { queueType = Bot - identifier = user.Id + identifier = user.Username + "#" + user.Discrim } if queueType == Bearer { From 1fe163f8341a95c8d5bab368fcba885d7ea24c75 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 18:47:01 +0530 Subject: [PATCH 15/27] Update lib/discord.go Co-authored-by: Germano Eichenberg --- lib/discord.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index dae55c3..a56fc94 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -71,9 +71,7 @@ func init() { wsProxy = os.Getenv("WS_PROXY") } - if os.Getenv("RATELIMIT_OVER_408") != "" { - ratelimitOver408 = os.Getenv("RATELIMIT_OVER_408") == "true" - } + lib.EnvGetBool("RATELIMIT_OVER_408", false) } type BotGatewayResponse struct { From e67bf8b9e878866d35e3cff5ff4bd2f398c2d0c7 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 18:56:35 +0530 Subject: [PATCH 16/27] Update discord.go with antiraid changes This also includes endpoint rewrites which is useful for the use case of custom gateway endpoints --- lib/discord.go | 147 ++++++++++++++++++++++++++++++++++++------------- 1 file changed, 109 insertions(+), 38 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index a56fc94..404b249 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -11,6 +11,7 @@ import ( "net" "net/http" "os" + "path/filepath" "strconv" "strings" "time" @@ -28,20 +29,25 @@ var endpointCache = make(map[string]*Cache) var disableRestLimitDetection = false +// List of endpoints to cache and their expiry times +var useEndpointCache bool var cacheEndpoints = map[string]time.Duration{ - "/api/users/@me": 10 * time.Minute, - "/api/v9/users/@me": 10 * time.Minute, - "/api/v10/users/@me": 10 * time.Minute, - "/api/gateway": 60 * time.Minute, - "/api/v9/gateway": 60 * time.Minute, - "/api/v10/gateway": 60 * time.Minute, - "/api/gateway/bot": 30 * time.Minute, - "/api/v9/gateway/bot": 30 * time.Minute, - "/api/v10/gateway/bot": 30 * time.Minute, - "/api/v9/applications/@me": 5 * time.Minute, - "/api/v10/applications/@me": 5 * time.Minute, + "/api/users/@me": 10 * time.Minute, + "/api/v*/users/@me": 10 * time.Minute, + "/api/gateway": 60 * time.Minute, + "/api/v*/gateway": 60 * time.Minute, + "/api/gateway/*": 30 * time.Minute, + "/api/v*/gateway/*": 30 * time.Minute, + "/api/v*/applications/@me": 5 * time.Minute, } +// In some cases, we may want to transparently rewrite endpoints +// +// For example, when using a gateway proxy, the proxy may provide its own /api/gateway/bot endpoint +// +// This allows transparently rewriting the endpoint to the proxy's +var endpointRewrite = map[string]string{} + var wsProxy string var ratelimitOver408 bool @@ -61,6 +67,37 @@ func init() { os.Setenv("PORT", argSplit[1]) case "ratelimit-over-408": ratelimitOver408 = true + case "use-endpoint-cache": + useEndpointCache = true + case "cache-endpoints": + if argSplit[1] == "" { + continue + } + + if argSplit[1] == "false" { + cacheEndpoints = make(map[string]time.Duration) + } else { + var endpoints map[string]time.Duration + + err := json.Unmarshal([]byte(argSplit[1]), &endpoints) + + if err != nil { + logrus.Fatal("Failed to parse cache-endpoints: ", err) + } + + cacheEndpoints = endpoints + } + case "endpoint-rewrite": + for _, rewrite := range strings.Split(argSplit[1], ",") { + // split by '->' + rewriteSplit := strings.Split(rewrite, "@") + + if len(rewriteSplit) != 2 { + logrus.Fatal("Invalid endpoint rewrite: ", rewrite) + } + + endpointRewrite[rewriteSplit[0]] = rewriteSplit[1] + } default: logrus.Fatal("Unknown argument: ", argSplit[0]) } @@ -71,7 +108,9 @@ func init() { wsProxy = os.Getenv("WS_PROXY") } - lib.EnvGetBool("RATELIMIT_OVER_408", false) + if os.Getenv("RATELIMIT_OVER_408") != "" { + ratelimitOver408 = os.Getenv("RATELIMIT_OVER_408") == "true" + } } type BotGatewayResponse struct { @@ -265,19 +304,25 @@ func GetBotUser(token string) (*BotUserResponse, error) { func doDiscordReq(ctx context.Context, path string, method string, body io.ReadCloser, header http.Header, query string) (*http.Response, error) { identifier := ctx.Value("identifier") if identifier == nil { - // Queues always have an identifier, if there's none in the context, we called the method from outside a queue - identifier = "Internal" + identifier = "internal" } - logger.Info(method, path+"?"+query) + logger.Info(method, " ", path+"?"+query) identifierStr, ok := identifier.(string) if ok { - // Check endpoint cache - if endpointCache[identifierStr] != nil { - cacheEntry := endpointCache[identifierStr].Get(path) + if useEndpointCache { + cache, ok := endpointCache[identifierStr] + + if !ok { + endpointCache[identifierStr] = NewCache() + cache = endpointCache[identifierStr] + } + // Check endpoint cache + cacheEntry := cache.Get(path) + if cacheEntry != nil { // Send cached response logger.WithFields(logrus.Fields{ @@ -285,27 +330,35 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC "path": path, "status": "200 (cached)", }).Debug("Discord request") - + headers := cacheEntry.Headers.Clone() headers.Set("X-Cached", "true") - + // Set rl headers so bot won't be perpetually stuck headers.Set("X-RateLimit-Limit", "5") headers.Set("X-RateLimit-Remaining", "5") headers.Set("X-RateLimit-Bucket", "cache") - + return &http.Response{ StatusCode: 200, Body: io.NopCloser(bytes.NewBuffer(cacheEntry.Data)), Header: headers, }, nil } - } else { - endpointCache[identifierStr] = NewCache() } } - discordReq, err := http.NewRequestWithContext(ctx, method, "https://discord.com"+path+"?"+query, body) + // Check for a rewrite + var urlBase = "https://discord.com" + for rw := range endpointRewrite { + if ok, _ := filepath.Match(rw, path); ok { + urlBase = endpointRewrite[rw] + break + + } + } + + discordReq, err := http.NewRequestWithContext(ctx, method, urlBase+path+"?"+query, body) if err != nil { return nil, err } @@ -329,8 +382,18 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC RequestHistogram.With(map[string]string{"route": route, "status": status, "method": method, "clientId": identifier.(string)}).Observe(elapsed) } - if wsProxy != "" { - if strings.HasSuffix(path, "/gateway") || strings.HasSuffix(path, "/gateway/bot") { + if wsProxy != "" && discordResp.StatusCode == 200 { + var isGwProxyUrl bool + + if path == "/api/gateway" || path == "/api/gateway/bot" { + isGwProxyUrl = true + } else if ok, _ := filepath.Match("/api/v*/gateway/bot", path); ok { + isGwProxyUrl = true + } else if ok, _ := filepath.Match("/api/v*/gateway", path); ok { + isGwProxyUrl = true + } + + if isGwProxyUrl { var data map[string]any err := json.NewDecoder(discordResp.Body).Decode(&data) @@ -351,21 +414,28 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC } } - if expiry, ok := cacheEndpoints[path]; ok { - if discordResp.StatusCode == 200 { - body, _ := io.ReadAll(discordResp.Body) - endpointCache[identifierStr].Set(path, &CacheEntry{ - Data: body, - CreatedAt: time.Now(), - ExpiresIn: &expiry, - Headers: discordResp.Header, - }) - - // Put body back into response - discordResp.Body = io.NopCloser(bytes.NewBuffer(body)) + var expiry *time.Duration + + for endpoint, exp := range cacheEndpoints { + if ok, _ := filepath.Match(endpoint, path); ok { + expiry = &exp + break } } + if expiry != nil && discordResp.StatusCode == 200 { + body, _ := io.ReadAll(discordResp.Body) + endpointCache[identifierStr].Set(path, &CacheEntry{ + Data: body, + CreatedAt: time.Now(), + ExpiresIn: expiry, + Headers: discordResp.Header, + }) + + // Put body back into response + discordResp.Body = io.NopCloser(bytes.NewBuffer(body)) + } + return discordResp, err } @@ -426,3 +496,4 @@ func ProcessRequest(ctx context.Context, item *QueueItem) (*http.Response, error return discordResp, nil } + From ffb558795d25582f98e46ffda9464e98fea95830 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 18:58:57 +0530 Subject: [PATCH 17/27] Update discord.go --- lib/discord.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index 404b249..022be8f 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -65,11 +65,11 @@ func init() { wsProxy = argSplit[1] case "port": os.Setenv("PORT", argSplit[1]) - case "ratelimit-over-408": + case "ratelimit-over-408": ratelimitOver408 = true - case "use-endpoint-cache": + case "use-endpoint-cache": useEndpointCache = true - case "cache-endpoints": + case "cache-endpoints": if argSplit[1] == "" { continue } @@ -108,9 +108,7 @@ func init() { wsProxy = os.Getenv("WS_PROXY") } - if os.Getenv("RATELIMIT_OVER_408") != "" { - ratelimitOver408 = os.Getenv("RATELIMIT_OVER_408") == "true" - } + ratelimitOver408 = lib.EnvGetBool("RATELIMIT_OVER_408", false) } type BotGatewayResponse struct { From 7c10e1791c82743ff4a2f572a57417e482cf6203 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:00:14 +0530 Subject: [PATCH 18/27] Update lib/discord.go Co-authored-by: Germano Eichenberg --- lib/discord.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index 022be8f..ea0a720 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -104,9 +104,7 @@ func init() { } } - if os.Getenv("WS_PROXY") != "" { - wsProxy = os.Getenv("WS_PROXY") - } + lib.EnvGet("WS_PROXY", "") ratelimitOver408 = lib.EnvGetBool("RATELIMIT_OVER_408", false) } From 4e8fed9aead8d76c68f2eeed3ec5be5e22565c82 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:01:14 +0530 Subject: [PATCH 19/27] Update discord.go --- lib/discord.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index ea0a720..5302ebe 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -104,7 +104,9 @@ func init() { } } - lib.EnvGet("WS_PROXY", "") + if wsProxy == "" { + wsProxy = lib.EnvGet("WS_PROXY", "") + } ratelimitOver408 = lib.EnvGetBool("RATELIMIT_OVER_408", false) } From 39e8929455786e05f8a94058ef132fbfefcf0e18 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:03:29 +0530 Subject: [PATCH 20/27] Update discord.go --- lib/discord.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index 5302ebe..f34986c 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -310,7 +310,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC identifierStr, ok := identifier.(string) if ok { - if useEndpointCache { + if useEndpointCache && identifier != "internal" { cache, ok := endpointCache[identifierStr] if !ok { From 6c0b85cc5cebacf2aa11787bf426198e527f8dd2 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:07:44 +0530 Subject: [PATCH 21/27] Update discord.go --- lib/discord.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index f34986c..cc8f69d 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -108,7 +108,13 @@ func init() { wsProxy = lib.EnvGet("WS_PROXY", "") } - ratelimitOver408 = lib.EnvGetBool("RATELIMIT_OVER_408", false) + if !ratelimitOver408 { + ratelimitOver408 = lib.EnvGetBool("RATELIMIT_OVER_408", false) + } + + if !useEndpointCache { + useEndpointCache = lib.EnvGetBool("USE_ENDPOINT_CACHE", false) + } } type BotGatewayResponse struct { From 18258635c8ce0619510466f587f054c8bd507f4f Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:09:03 +0530 Subject: [PATCH 22/27] Update cache.go --- lib/cache.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/cache.go b/lib/cache.go index 60997e1..378105c 100644 --- a/lib/cache.go +++ b/lib/cache.go @@ -8,15 +8,12 @@ import ( type CacheEntry struct { Data []byte CreatedAt time.Time - ExpiresIn *time.Duration + ExpiresIn time.Duration Headers http.Header } func (c *CacheEntry) Expired() bool { - if c.ExpiresIn == nil { - return false - } - return time.Since(c.CreatedAt) > *c.ExpiresIn + return time.Since(c.CreatedAt) > c.ExpiresIn } type Cache struct { From eb33179db024b6e80bc156560ae57139b55569d1 Mon Sep 17 00:00:00 2001 From: Sanandan Sashikumar Date: Tue, 25 Jun 2024 13:40:14 +0000 Subject: [PATCH 23/27] fix compile errors --- lib/discord.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index cc8f69d..a222e7d 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -105,15 +105,15 @@ func init() { } if wsProxy == "" { - wsProxy = lib.EnvGet("WS_PROXY", "") + wsProxy = EnvGet("WS_PROXY", "") } if !ratelimitOver408 { - ratelimitOver408 = lib.EnvGetBool("RATELIMIT_OVER_408", false) + ratelimitOver408 = EnvGetBool("RATELIMIT_OVER_408", false) } if !useEndpointCache { - useEndpointCache = lib.EnvGetBool("USE_ENDPOINT_CACHE", false) + useEndpointCache = EnvGetBool("USE_ENDPOINT_CACHE", false) } } @@ -432,7 +432,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC endpointCache[identifierStr].Set(path, &CacheEntry{ Data: body, CreatedAt: time.Now(), - ExpiresIn: expiry, + ExpiresIn: *expiry, Headers: discordResp.Header, }) From c04ae68b4b21a7c86519e9a178b8ad72d7f39afe Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:15:55 +0530 Subject: [PATCH 24/27] Delete Makefile Not used by upstream --- Makefile | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 2d79bcd..0000000 --- a/Makefile +++ /dev/null @@ -1,2 +0,0 @@ -all: - CGO_ENABLED=0 go build -v From 401676c318590ad58ce7f45e6888a59aecd23ace Mon Sep 17 00:00:00 2001 From: Rootspring Date: Tue, 25 Jun 2024 19:22:42 +0530 Subject: [PATCH 25/27] Update discord.go --- lib/discord.go | 36 +++++++++++++++++++----------------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index a222e7d..669d1f3 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -418,26 +418,28 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC } } - var expiry *time.Duration + if useEndpointCache { + var expiry *time.Duration - for endpoint, exp := range cacheEndpoints { - if ok, _ := filepath.Match(endpoint, path); ok { - expiry = &exp - break + for endpoint, exp := range cacheEndpoints { + if ok, _ := filepath.Match(endpoint, path); ok { + expiry = &exp + break + } } - } - if expiry != nil && discordResp.StatusCode == 200 { - body, _ := io.ReadAll(discordResp.Body) - endpointCache[identifierStr].Set(path, &CacheEntry{ - Data: body, - CreatedAt: time.Now(), - ExpiresIn: *expiry, - Headers: discordResp.Header, - }) - - // Put body back into response - discordResp.Body = io.NopCloser(bytes.NewBuffer(body)) + if expiry != nil && discordResp.StatusCode == 200 { + body, _ := io.ReadAll(discordResp.Body) + endpointCache[identifierStr].Set(path, &CacheEntry{ + Data: body, + CreatedAt: time.Now(), + ExpiresIn: *expiry, + Headers: discordResp.Header, + }) + + // Put body back into response + discordResp.Body = io.NopCloser(bytes.NewBuffer(body)) + } } return discordResp, err From 4d8989e0a602212e403a70d877e5ad8fb0f6293a Mon Sep 17 00:00:00 2001 From: Rootspring Date: Sat, 29 Jun 2024 10:11:49 +0530 Subject: [PATCH 26/27] Update lib/discord.go Co-authored-by: Germano Eichenberg --- lib/discord.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/discord.go b/lib/discord.go index 669d1f3..73cdd24 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -389,11 +389,7 @@ func doDiscordReq(ctx context.Context, path string, method string, body io.ReadC if wsProxy != "" && discordResp.StatusCode == 200 { var isGwProxyUrl bool - if path == "/api/gateway" || path == "/api/gateway/bot" { - isGwProxyUrl = true - } else if ok, _ := filepath.Match("/api/v*/gateway/bot", path); ok { - isGwProxyUrl = true - } else if ok, _ := filepath.Match("/api/v*/gateway", path); ok { + if strings.HasSuffix(path, "/gateway") || strings.HasSuffix(path, "/gateway/bot") { isGwProxyUrl = true } From 8f8602792f62923aa55ac1c1180755a56cbd8bc3 Mon Sep 17 00:00:00 2001 From: Rootspring Date: Sat, 29 Jun 2024 10:12:04 +0530 Subject: [PATCH 27/27] Update lib/discord.go Co-authored-by: Germano Eichenberg --- lib/discord.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/discord.go b/lib/discord.go index 73cdd24..74b0904 100644 --- a/lib/discord.go +++ b/lib/discord.go @@ -308,7 +308,7 @@ func GetBotUser(token string) (*BotUserResponse, error) { func doDiscordReq(ctx context.Context, path string, method string, body io.ReadCloser, header http.Header, query string) (*http.Response, error) { identifier := ctx.Value("identifier") if identifier == nil { - identifier = "internal" + identifier = "Internal" } logger.Info(method, " ", path+"?"+query)