From c80cde3f23dc4440591483eb3683cbef14db3eb2 Mon Sep 17 00:00:00 2001 From: Ghost Scripter Date: Fri, 27 Sep 2024 19:18:25 +0530 Subject: [PATCH] added rate limit with axios requests --- adapters/zerolend/package.json | 2 + adapters/zerolend/src/sdk/foxy.ts | 22 +++++++--- adapters/zerolend/src/sdk/lp.ts | 22 +++++++--- adapters/zerolend/src/sdk/stake.ts | 22 +++++++--- adapters/zerolend/src/sdk/tvl.ts | 22 +++++++--- adapters/zerolend/yarn.lock | 64 ++++++++++++++++++++++++++++++ 6 files changed, 130 insertions(+), 24 deletions(-) diff --git a/adapters/zerolend/package.json b/adapters/zerolend/package.json index 149ad500..a02ecafe 100644 --- a/adapters/zerolend/package.json +++ b/adapters/zerolend/package.json @@ -15,6 +15,8 @@ "author": "", "license": "UNLICENSED", "dependencies": { + "axios": "^1.7.7", + "axios-rate-limit": "^1.4.0", "csv-parser": "^3.0.0", "fast-csv": "^5.0.1" }, diff --git a/adapters/zerolend/src/sdk/foxy.ts b/adapters/zerolend/src/sdk/foxy.ts index cc6d1832..00357c1f 100644 --- a/adapters/zerolend/src/sdk/foxy.ts +++ b/adapters/zerolend/src/sdk/foxy.ts @@ -1,3 +1,5 @@ +import axios from "axios"; +import rateLimit from "axios-rate-limit"; import { BlockData, IUserReserve, @@ -5,6 +7,11 @@ import { OutputDataSchemaRow, } from "./types"; +const axiosInstance = rateLimit(axios.create(), { + maxRequests: 5, + perMilliseconds: 1000, +}); + const queryURL = "https://api.goldsky.com/api/public/project_clsk1wzatdsls01wchl2e4n0y/subgraphs/zerolend-linea-foxy/1.0.0/gn"; @@ -43,12 +50,15 @@ export const getUserTVLFoxyByBlock = async ( } }`; - const response = await fetch(queryURL, { - method: "POST", - body: JSON.stringify({ query }), - headers: { "Content-Type": "application/json" }, - }); - const batch: ILPResponse = await response.json(); + const response = await axiosInstance.post( + queryURL, + { query }, + { + headers: { "Content-Type": "application/json" }, + } + ); + + const batch: ILPResponse = await response.data; if (!batch.data || batch.data.userReserves.length == 0) break; diff --git a/adapters/zerolend/src/sdk/lp.ts b/adapters/zerolend/src/sdk/lp.ts index 19502e03..8b4a9e1d 100644 --- a/adapters/zerolend/src/sdk/lp.ts +++ b/adapters/zerolend/src/sdk/lp.ts @@ -1,3 +1,5 @@ +import axios from "axios"; +import rateLimit from "axios-rate-limit"; import { BlockData, IOmniStakingData, @@ -5,6 +7,11 @@ import { OutputDataSchemaRow, } from "./types"; +const axiosInstance = rateLimit(axios.create(), { + maxRequests: 5, + perMilliseconds: 1000, +}); + const queryURL = "https://api.goldsky.com/api/public/project_clsk1wzatdsls01wchl2e4n0y/subgraphs/zerolend-omnistaking/1.0.2/gn"; @@ -32,12 +39,15 @@ export const getUserLPByBlock = async ( } }`; - const response = await fetch(queryURL, { - method: "POST", - body: JSON.stringify({ query }), - headers: { "Content-Type": "application/json" }, - }); - const batch: IOmniStakingResponse = await response.json(); + const response = await axiosInstance.post( + queryURL, + { query }, + { + headers: { "Content-Type": "application/json" }, + } + ); + + const batch: IOmniStakingResponse = await response.data; if (!batch.data || batch.data.tokenBalances.length == 0) break; diff --git a/adapters/zerolend/src/sdk/stake.ts b/adapters/zerolend/src/sdk/stake.ts index 4f44abc7..45403042 100644 --- a/adapters/zerolend/src/sdk/stake.ts +++ b/adapters/zerolend/src/sdk/stake.ts @@ -1,3 +1,5 @@ +import axios from "axios"; +import rateLimit from "axios-rate-limit"; import { BlockData, IOmniStakingData, @@ -5,6 +7,11 @@ import { OutputDataSchemaRow, } from "./types"; +const axiosInstance = rateLimit(axios.create(), { + maxRequests: 5, + perMilliseconds: 1000, +}); + const queryURL = "https://api.goldsky.com/api/public/project_clsk1wzatdsls01wchl2e4n0y/subgraphs/zerolend-omnistaking/1.0.2/gn"; @@ -34,12 +41,15 @@ export const getUserStakeByBlock = async ( } }`; - const response = await fetch(queryURL, { - method: "POST", - body: JSON.stringify({ query }), - headers: { "Content-Type": "application/json" }, - }); - const batch: IOmniStakingResponse = await response.json(); + const response = await axiosInstance.post( + queryURL, + { query }, + { + headers: { "Content-Type": "application/json" }, + } + ); + + const batch: IOmniStakingResponse = await response.data; if (!batch.data || batch.data.tokenBalances.length == 0) break; diff --git a/adapters/zerolend/src/sdk/tvl.ts b/adapters/zerolend/src/sdk/tvl.ts index 909abe2f..3838c404 100644 --- a/adapters/zerolend/src/sdk/tvl.ts +++ b/adapters/zerolend/src/sdk/tvl.ts @@ -1,3 +1,5 @@ +import axios from "axios"; +import rateLimit from "axios-rate-limit"; import { BlockData, IUserReserve, @@ -5,6 +7,11 @@ import { OutputDataSchemaRow, } from "./types"; +const axiosInstance = rateLimit(axios.create(), { + maxRequests: 5, + perMilliseconds: 1000, +}); + const queryURL = "https://api.goldsky.com/api/public/project_clsk1wzatdsls01wchl2e4n0y/subgraphs/zerolend-linea/1.0.0/gn"; @@ -39,12 +46,15 @@ export const getUserTVLLegacyByBlock = async ( } }`; - const response = await fetch(queryURL, { - method: "POST", - body: JSON.stringify({ query }), - headers: { "Content-Type": "application/json" }, - }); - const batch: ILPResponse = await response.json(); + const response = await axiosInstance.post( + queryURL, + { query }, + { + headers: { "Content-Type": "application/json" }, + } + ); + + const batch: ILPResponse = await response.data; if (!batch.data || batch.data.userReserves.length == 0) break; diff --git a/adapters/zerolend/yarn.lock b/adapters/zerolend/yarn.lock index 1f64dc83..c556efac 100644 --- a/adapters/zerolend/yarn.lock +++ b/adapters/zerolend/yarn.lock @@ -32,6 +32,34 @@ dependencies: undici-types "~5.26.4" +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + +axios-rate-limit@^1.4.0: + version "1.4.0" + resolved "https://registry.yarnpkg.com/axios-rate-limit/-/axios-rate-limit-1.4.0.tgz#c8eab732cf22dd0d431d5b04d9fc1836b05c0ebd" + integrity sha512-uM5PbmSUdSle1I+59Av/wpLuNRobfatIR+FyylSoHcVHT20ohjflNnLMEHZQr7N2QVG/Wlt8jekIPhWwoKtpXQ== + dependencies: + axios ">=0.18.0" + +axios@>=0.18.0, axios@^1.7.7: + version "1.7.7" + resolved "https://registry.yarnpkg.com/axios/-/axios-1.7.7.tgz#2f554296f9892a72ac8d8e4c5b79c14a91d0a47f" + integrity sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q== + dependencies: + follow-redirects "^1.15.6" + form-data "^4.0.0" + proxy-from-env "^1.1.0" + +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + csv-parser@^3.0.0: version "3.0.0" resolved "https://registry.npmjs.org/csv-parser/-/csv-parser-3.0.0.tgz" @@ -39,6 +67,11 @@ csv-parser@^3.0.0: dependencies: minimist "^1.2.0" +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + fast-csv@^5.0.1: version "5.0.1" resolved "https://registry.npmjs.org/fast-csv/-/fast-csv-5.0.1.tgz" @@ -47,6 +80,20 @@ fast-csv@^5.0.1: "@fast-csv/format" "5.0.0" "@fast-csv/parse" "5.0.0" +follow-redirects@^1.15.6: + version "1.15.9" + resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.15.9.tgz#a604fa10e443bf98ca94228d9eebcc2e8a2c8ee1" + integrity sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ== + +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + lodash.escaperegexp@^4.1.2: version "4.1.2" resolved "https://registry.npmjs.org/lodash.escaperegexp/-/lodash.escaperegexp-4.1.2.tgz" @@ -87,11 +134,28 @@ lodash.uniq@^4.5.0: resolved "https://registry.npmjs.org/lodash.uniq/-/lodash.uniq-4.5.0.tgz" integrity sha512-xfBaXQd9ryd9dlSDvnvI0lvxfLJlYAZzXomUYzLKtUeOQvOP5piqAWuGtrhWeqaXK9hhoM/iyJc5AV+XfsX3HQ== +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + minimist@^1.2.0: version "1.2.8" resolved "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz" integrity sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA== +proxy-from-env@^1.1.0: + version "1.1.0" + resolved "https://registry.yarnpkg.com/proxy-from-env/-/proxy-from-env-1.1.0.tgz#e102f16ca355424865755d2c9e8ea4f24d58c3e2" + integrity sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg== + typescript@^5.3.3: version "5.4.5" resolved "https://registry.npmjs.org/typescript/-/typescript-5.4.5.tgz"