Add proxy location (#934)

* fix: update prisma setups for proxy locations

* fix: set location optional add update dto

* feat: use proxy for specified location if defined in adapter

* fix: remove unused import

* fix: fix test code

* fix: move location setting into definition

* fix: check location existance

* feat: update based on feedbacks

* docs: update readme

api/README.md

@@ -132,3 +132,10 @@ GET http://localhost:3000/api/v1/proxy
 3. Insert `Aggregator` (initial settings)
 4. Insert `Proxy` (during fetching data with Orakl Network Fetcher)
 5. Insert `Data` (during regular data fetching with Orakl Network Fetcher)
+
+## Proxy Location codes
+
+| location  | code |
+| --------- | ---- |
+| Korea     | kr   |
+| Singapore | sg   |

api/prisma/migrations/20231116075406_add_proxy_location/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `location` to the `proxies` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "proxies" ADD COLUMN     "location" TEXT NOT NULL;

api/prisma/migrations/20231116080058_set_proxy_location_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "proxies" ALTER COLUMN "location" DROP NOT NULL;

api/prisma/schema.prisma

@@ -148,10 +148,11 @@ model Error {
 }
 
 model Proxy {
-  id       BigInt @id @default(autoincrement())
+  id       BigInt  @id @default(autoincrement())
   protocol String
   host     String
   port     Int
+  location String?
 
   @@unique([protocol, host, port])
   @@map("proxies")

api/src/proxy/dto/proxy.ts

@@ -9,4 +9,7 @@ export class ProxyDto {
 
   @ApiProperty()
   port: number
+
+  @ApiProperty()
+  location?: string
 }

api/src/proxy/proxy.service.spec.ts

@@ -29,13 +29,26 @@ describe('ProxyService', () => {
     const proxyData = {
       protocol: 'http',
       host: '127.0.0.1',
-      port: 80
+      port: 80,
+      location: 'kr'
     }
     const proxyObj = await proxy.create(proxyData)
     expect(proxyObj.protocol).toBe(proxyData.protocol)
     expect(proxyObj.host).toBe(proxyData.host)
     expect(proxyObj.port).toBe(proxyData.port)
+    expect(proxyObj.location).toBe(proxyData.location)
+
+    const proxyDataWithoutLocation = {
+      protocol: 'http',
+      host: '127.0.0.2',
+      port: 80
+    }
 
+    const proxyObjWithoutLocation = await proxy.create(proxyDataWithoutLocation)
+    expect(proxyObjWithoutLocation.protocol).toBe(proxyDataWithoutLocation.protocol)
+    expect(proxyObjWithoutLocation.host).toBe(proxyDataWithoutLocation.host)
+    expect(proxyObjWithoutLocation.port).toBe(proxyDataWithoutLocation.port)
+    expect(proxyObjWithoutLocation.location).toBe(null)
     // The same proxy cannot be defined twice
     await expect(async () => {
       await proxy.create(proxyData)

cli/src/proxy.ts

@@ -1,6 +1,6 @@
 import axios from 'axios'
 import { command, subcommands, option, string as cmdstring, number } from 'cmd-ts'
-import { idOption, buildUrl, isOraklNetworkApiHealthy } from './utils'
+import { idOption, buildUrl, isOraklNetworkApiHealthy, proxyOptionalOption } from './utils'
 import { ORAKL_NETWORK_API_URL } from './settings'
 
 const PROXY_ENDPOINT = buildUrl(ORAKL_NETWORK_API_URL, 'proxy')
@@ -30,7 +30,8 @@ export function proxySub() {
       port: option({
         type: number,
         long: 'port'
-      })
+      }),
+      location: proxyOptionalOption
     },
     handler: insertHandler()
   })
@@ -70,16 +71,18 @@ export function insertHandler() {
   async function wrapper({
     protocol,
     host,
-    port
+    port,
+    location
   }: {
     protocol: string
     host: string
     port: number
+    location?: string
   }) {
     if (!(await isOraklNetworkApiHealthy())) return
 
     try {
-      const response = (await axios.post(PROXY_ENDPOINT, { protocol, host, port }))?.data
+      const response = (await axios.post(PROXY_ENDPOINT, { protocol, host, port, location }))?.data
       console.dir(response, { depth: null })
     } catch (e) {
       console.error('Proxy was not inserted. Reason:')

cli/src/utils.ts

@@ -18,6 +18,11 @@ export const fetcherTypeOptionalOption = option({
   long: 'fetcherType'
 })
 
+export const proxyOptionalOption = option({
+  type: optional(cmdstring),
+  long: 'location'
+})
+
 export const idOption = option({
   type: cmdnumber,
   long: 'id'

fetcher/src/job/job.types.ts

@@ -78,4 +78,5 @@ export interface IProxy {
   protocol: string | undefined
   host: string | undefined
   port: number | undefined
+  location?: string | undefined
 }

fetcher/src/job/job.utils.ts

@@ -180,7 +180,18 @@ export function extractFeeds(
   const feeds = adapter.feeds.map((f) => {
     let proxy: IProxy
     try {
-      proxy = proxySelector(f.definition.url)
+      if (!f.definition.location) {
+        proxy = proxySelector(f.definition.url)
+      } else {
+        const availableProxies = proxies.filter(
+          (item) => item.location && item.location === f.definition.location
+        )
+        if (availableProxies.length == 0) {
+          throw `no proxies available for location:${f.definition.location}`
+        }
+        const randomIndex = Math.floor(Math.random() * availableProxies.length)
+        proxy = availableProxies[randomIndex]
+      }
     } catch (e) {
       logger.error('Assigning proxy has failed')
       logger.error(e)