Merge branch 'master' into 887-listener-worker-reporter-for-l2-vrfrr

.github/workflows/core.image+upload.yaml

@@ -6,6 +6,7 @@ on:
       - master
     paths:
       - "core/**"
+  workflow_dispatch:
 
 env:
   ecr_url: public.ecr.aws/bisonai/orakl-core
@@ -50,6 +51,13 @@ jobs:
     steps:
       - uses: actions/checkout@v3
 
+      - name: Replace symlink with real files
+        run: |
+         rm -f ./core/src/por/job.errors.ts ./core/src/por/job.types.ts ./core/src/por/reducer.ts
+         cp ./fetcher/src/job/job.errors.ts ./core/src/por/job.errors.ts
+         cp ./fetcher/src/job/job.types.ts ./core/src/por/job.types.ts
+         cp ./fetcher/src/job/job.reducer.ts ./core/src/por/reducer.ts
+
       - name: Docker build orakl-core
         run:  docker-compose -f core/docker-compose.build.yaml build
 

api/README.md

@@ -132,3 +132,10 @@ GET http://localhost:3000/api/v1/proxy
 3. Insert `Aggregator` (initial settings)
 4. Insert `Proxy` (during fetching data with Orakl Network Fetcher)
 5. Insert `Data` (during regular data fetching with Orakl Network Fetcher)
+
+## Proxy Location codes
+
+| location  | code |
+| --------- | ---- |
+| Korea     | kr   |
+| Singapore | sg   |

api/prisma/migrations/20231116075406_add_proxy_location/migration.sql

@@ -0,0 +1,8 @@
+/*
+  Warnings:
+
+  - Added the required column `location` to the `proxies` table without a default value. This is not possible if the table is not empty.
+
+*/
+-- AlterTable
+ALTER TABLE "proxies" ADD COLUMN     "location" TEXT NOT NULL;

api/prisma/migrations/20231116080058_set_proxy_location_optional/migration.sql

@@ -0,0 +1,2 @@
+-- AlterTable
+ALTER TABLE "proxies" ALTER COLUMN "location" DROP NOT NULL;

api/prisma/schema.prisma

@@ -148,10 +148,11 @@ model Error {
 }
 
 model Proxy {
-  id       BigInt @id @default(autoincrement())
+  id       BigInt  @id @default(autoincrement())
   protocol String
   host     String
   port     Int
+  location String?
 
   @@unique([protocol, host, port])
   @@map("proxies")

api/src/aggregate/aggregate.service.ts

@@ -61,13 +61,16 @@ export class AggregateService {
    */
   async findLatest(latestAggregateDto: LatestAggregateDto) {
     const { aggregatorHash } = latestAggregateDto
-    return await this.prisma.aggregate.findFirst({
-      where: { aggregator: { aggregatorHash } },
-      orderBy: [
-        {
-          timestamp: 'desc'
-        }
-      ]
-    })
+    const query = Prisma.sql`SELECT aggregate_id as id, timestamp, value, aggregator_id as "aggregatorId"
+      FROM aggregates
+      WHERE aggregator_id = (SELECT aggregator_id FROM aggregators WHERE aggregator_hash = ${aggregatorHash})
+      ORDER BY timestamp DESC
+      LIMIT 1;`
+    const result: Prisma.AggregateScalarFieldEnum[] = await this.prisma.$queryRaw(query)
+    if (result.length == 1) {
+      return result[0]
+    } else {
+      throw Error(`Expected one row. Received ${result.length}`)
+    }
   }
 }

api/src/proxy/dto/proxy.ts

@@ -9,4 +9,7 @@ export class ProxyDto {
 
   @ApiProperty()
   port: number
+
+  @ApiProperty()
+  location?: string
 }

api/src/proxy/proxy.service.spec.ts

@@ -29,13 +29,26 @@ describe('ProxyService', () => {
     const proxyData = {
       protocol: 'http',
       host: '127.0.0.1',
-      port: 80
+      port: 80,
+      location: 'kr'
     }
     const proxyObj = await proxy.create(proxyData)
     expect(proxyObj.protocol).toBe(proxyData.protocol)
     expect(proxyObj.host).toBe(proxyData.host)
     expect(proxyObj.port).toBe(proxyData.port)
+    expect(proxyObj.location).toBe(proxyData.location)
+
+    const proxyDataWithoutLocation = {
+      protocol: 'http',
+      host: '127.0.0.2',
+      port: 80
+    }
 
+    const proxyObjWithoutLocation = await proxy.create(proxyDataWithoutLocation)
+    expect(proxyObjWithoutLocation.protocol).toBe(proxyDataWithoutLocation.protocol)
+    expect(proxyObjWithoutLocation.host).toBe(proxyDataWithoutLocation.host)
+    expect(proxyObjWithoutLocation.port).toBe(proxyDataWithoutLocation.port)
+    expect(proxyObjWithoutLocation.location).toBe(null)
     // The same proxy cannot be defined twice
     await expect(async () => {
       await proxy.create(proxyData)

cli/src/proxy.ts

@@ -1,6 +1,6 @@
 import axios from 'axios'
 import { command, subcommands, option, string as cmdstring, number } from 'cmd-ts'
-import { idOption, buildUrl, isOraklNetworkApiHealthy } from './utils'
+import { idOption, buildUrl, isOraklNetworkApiHealthy, proxyOptionalOption } from './utils'
 import { ORAKL_NETWORK_API_URL } from './settings'
 
 const PROXY_ENDPOINT = buildUrl(ORAKL_NETWORK_API_URL, 'proxy')
@@ -30,7 +30,8 @@ export function proxySub() {
       port: option({
         type: number,
         long: 'port'
-      })
+      }),
+      location: proxyOptionalOption
     },
     handler: insertHandler()
   })
@@ -70,16 +71,18 @@ export function insertHandler() {
   async function wrapper({
     protocol,
     host,
-    port
+    port,
+    location
   }: {
     protocol: string
     host: string
     port: number
+    location?: string
   }) {
     if (!(await isOraklNetworkApiHealthy())) return
 
     try {
-      const response = (await axios.post(PROXY_ENDPOINT, { protocol, host, port }))?.data
+      const response = (await axios.post(PROXY_ENDPOINT, { protocol, host, port, location }))?.data
       console.dir(response, { depth: null })
     } catch (e) {
       console.error('Proxy was not inserted. Reason:')

cli/src/utils.ts

@@ -18,6 +18,11 @@ export const fetcherTypeOptionalOption = option({
   long: 'fetcherType'
 })
 
+export const proxyOptionalOption = option({
+  type: optional(cmdstring),
+  long: 'location'
+})
+
 export const idOption = option({
   type: cmdnumber,
   long: 'id'

core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bisonai/orakl-core",
-  "version": "0.5.0",
+  "version": "0.5.1",
   "type": "module",
   "description": "The Orakl Network Core",
   "files": [

core/src/errors.ts

@@ -61,5 +61,16 @@ export enum OraklErrorCode {
   CaverTxTransactionFailed,
   DelegatorServerIssue,
   FailedInsertData,
-  FailedInsertAggregatedData
+  FailedInsertAggregatedData,
+  AxiosBadOptionValue,
+  AxiosBadOption,
+  AxiosTimeOut,
+  AxiosNetworkError,
+  AxiosTooManyRedirects,
+  AxiosDeprecated,
+  AxiosBadResponse,
+  AxiosBadRequest,
+  AxiosCanceledByUser,
+  AxiosNotSupported,
+  AxiosInvalidUrl
 }

core/src/reporter/utils.ts

@@ -4,9 +4,9 @@ import Caver from 'caver-js'
 import { ethers } from 'ethers'
 import { Logger } from 'pino'
 import { OraklError, OraklErrorCode } from '../errors'
-import { ORAKL_NETWORK_DELEGATOR_URL } from '../settings'
+import { DELEGATOR_TIMEOUT, ORAKL_NETWORK_DELEGATOR_URL } from '../settings'
 import { ITransactionData } from '../types'
-import { add0x, buildUrl } from '../utils'
+import { add0x, buildUrl, getOraklErrorCode } from '../utils'
 
 const FILE_NAME = import.meta.url
 
@@ -176,15 +176,14 @@ export async function sendTransactionDelegatedFee({
   const endpoint = buildUrl(ORAKL_NETWORK_DELEGATOR_URL, `sign`)
 
   let response
+
   try {
-    response = (
-      await axios.post(endpoint, {
-        ...transactionData
-      })
-    )?.data
+    response = (await axios.post(endpoint, { ...transactionData }, { timeout: DELEGATOR_TIMEOUT }))
+      ?.data
     _logger.debug(response)
   } catch (e) {
-    throw new OraklError(OraklErrorCode.DelegatorServerIssue)
+    const errorCode = getOraklErrorCode(e, OraklErrorCode.DelegatorServerIssue)
+    throw new OraklError(errorCode)
   }
 
   try {

core/src/settings.ts

@@ -7,6 +7,8 @@ export const ORAKL_NETWORK_API_URL =
 export const ORAKL_NETWORK_DELEGATOR_URL =
   process.env.ORAKL_NETWORK_DELEGATOR_URL || 'http://localhost:3002/api/v1'
 
+export const DELEGATOR_TIMEOUT = Number(process.env.DELEGATOR_TIMEOUT) || 3000
+
 export const DEPLOYMENT_NAME = process.env.DEPLOYMENT_NAME || 'orakl'
 export const NODE_ENV = process.env.NODE_ENV
 export const HEALTH_CHECK_PORT = process.env.HEALTH_CHECK_PORT

core/src/utils.ts

@@ -4,6 +4,7 @@ import * as Fs from 'node:fs/promises'
 import os from 'node:os'
 import type { RedisClientType } from 'redis'
 import { createClient } from 'redis'
+import { OraklErrorCode } from './errors'
 import { SLACK_WEBHOOK_URL } from './settings'
 
 export async function loadJson(filepath) {
@@ -123,3 +124,32 @@ export function buildUrl(host: string, path: string) {
   const url = [host, path].join('/')
   return url.replace(/([^:]\/)\/+/g, '$1')
 }
+
+// axios errors are defined in official repo (https://github.com/axios/axios#error-types)
+export const getOraklErrorCode = (e, defaultErrorCode) => {
+  if (e.code == 'ERR_BAD_OPTION_VALUE') {
+    return OraklErrorCode.AxiosBadOptionValue
+  } else if (e.code == 'ERR_BAD_OPTION') {
+    return OraklErrorCode.AxiosBadOption
+  } else if (e.code == 'ECONNABORTED' || e.code == 'ETIMEDOUT') {
+    return OraklErrorCode.AxiosTimeOut
+  } else if (e.code == 'ERR_NETWORK') {
+    return OraklErrorCode.AxiosNetworkError
+  } else if (e.code == 'ERR_FR_TOO_MANY_REDIRECTS') {
+    return OraklErrorCode.AxiosTooManyRedirects
+  } else if (e.code == 'ERR_DEPRECATED') {
+    return OraklErrorCode.AxiosDeprecated
+  } else if (e.code == 'ERR_BAD_RESPONSE') {
+    return OraklErrorCode.AxiosBadResponse
+  } else if (e.code == 'ERR_BAD_REQUEST') {
+    return OraklErrorCode.AxiosBadRequest
+  } else if (e.code == 'ERR_CANCELED') {
+    return OraklErrorCode.AxiosCanceledByUser
+  } else if (e.code == 'ERR_NOT_SUPPORT') {
+    return OraklErrorCode.AxiosNotSupported
+  } else if (e.code == 'ERR_INVALID_URL') {
+    return OraklErrorCode.AxiosInvalidUrl
+  } else {
+    return defaultErrorCode
+  }
+}

fetcher/src/job/job.types.ts

@@ -78,4 +78,5 @@ export interface IProxy {
   protocol: string | undefined
   host: string | undefined
   port: number | undefined
+  location?: string | undefined
 }

fetcher/src/job/job.utils.ts

@@ -180,7 +180,18 @@ export function extractFeeds(
   const feeds = adapter.feeds.map((f) => {
     let proxy: IProxy
     try {
-      proxy = proxySelector(f.definition.url)
+      if (!f.definition.location) {
+        proxy = proxySelector(f.definition.url)
+      } else {
+        const availableProxies = proxies.filter(
+          (item) => item.location && item.location === f.definition.location
+        )
+        if (availableProxies.length == 0) {
+          throw `no proxies available for location:${f.definition.location}`
+        }
+        const randomIndex = Math.floor(Math.random() * availableProxies.length)
+        proxy = availableProxies[randomIndex]
+      }
     } catch (e) {
       logger.error('Assigning proxy has failed')
       logger.error(e)