diff --git a/.github/workflows/core.image+upload.yaml b/.github/workflows/core.image+upload.yaml index 02d459289..6bef062fa 100644 --- a/.github/workflows/core.image+upload.yaml +++ b/.github/workflows/core.image+upload.yaml @@ -6,6 +6,7 @@ on: - master paths: - "core/**" + workflow_dispatch: env: ecr_url: public.ecr.aws/bisonai/orakl-core @@ -50,6 +51,13 @@ jobs: steps: - uses: actions/checkout@v3 + - name: Replace symlink with real files + run: | + rm -f ./core/src/por/job.errors.ts ./core/src/por/job.types.ts ./core/src/por/reducer.ts + cp ./fetcher/src/job/job.errors.ts ./core/src/por/job.errors.ts + cp ./fetcher/src/job/job.types.ts ./core/src/por/job.types.ts + cp ./fetcher/src/job/job.reducer.ts ./core/src/por/reducer.ts + - name: Docker build orakl-core run: docker-compose -f core/docker-compose.build.yaml build diff --git a/api/README.md b/api/README.md index c0796fb62..ca5f048a3 100644 --- a/api/README.md +++ b/api/README.md @@ -132,3 +132,10 @@ GET http://localhost:3000/api/v1/proxy 3. Insert `Aggregator` (initial settings) 4. Insert `Proxy` (during fetching data with Orakl Network Fetcher) 5. Insert `Data` (during regular data fetching with Orakl Network Fetcher) + +## Proxy Location codes + +| location | code | +| --------- | ---- | +| Korea | kr | +| Singapore | sg | diff --git a/api/prisma/migrations/20231116075406_add_proxy_location/migration.sql b/api/prisma/migrations/20231116075406_add_proxy_location/migration.sql new file mode 100644 index 000000000..15d8c65b4 --- /dev/null +++ b/api/prisma/migrations/20231116075406_add_proxy_location/migration.sql @@ -0,0 +1,8 @@ +/* + Warnings: + + - Added the required column `location` to the `proxies` table without a default value. This is not possible if the table is not empty. + +*/ +-- AlterTable +ALTER TABLE "proxies" ADD COLUMN "location" TEXT NOT NULL; \ No newline at end of file diff --git a/api/prisma/migrations/20231116080058_set_proxy_location_optional/migration.sql b/api/prisma/migrations/20231116080058_set_proxy_location_optional/migration.sql new file mode 100644 index 000000000..9e3f58fcd --- /dev/null +++ b/api/prisma/migrations/20231116080058_set_proxy_location_optional/migration.sql @@ -0,0 +1,2 @@ +-- AlterTable +ALTER TABLE "proxies" ALTER COLUMN "location" DROP NOT NULL; diff --git a/api/prisma/schema.prisma b/api/prisma/schema.prisma index a40bc6118..c18f9b796 100644 --- a/api/prisma/schema.prisma +++ b/api/prisma/schema.prisma @@ -148,10 +148,11 @@ model Error { } model Proxy { - id BigInt @id @default(autoincrement()) + id BigInt @id @default(autoincrement()) protocol String host String port Int + location String? @@unique([protocol, host, port]) @@map("proxies") diff --git a/api/src/aggregate/aggregate.service.ts b/api/src/aggregate/aggregate.service.ts index 878ec9f35..efa688fe7 100644 --- a/api/src/aggregate/aggregate.service.ts +++ b/api/src/aggregate/aggregate.service.ts @@ -61,13 +61,16 @@ export class AggregateService { */ async findLatest(latestAggregateDto: LatestAggregateDto) { const { aggregatorHash } = latestAggregateDto - return await this.prisma.aggregate.findFirst({ - where: { aggregator: { aggregatorHash } }, - orderBy: [ - { - timestamp: 'desc' - } - ] - }) + const query = Prisma.sql`SELECT aggregate_id as id, timestamp, value, aggregator_id as "aggregatorId" + FROM aggregates + WHERE aggregator_id = (SELECT aggregator_id FROM aggregators WHERE aggregator_hash = ${aggregatorHash}) + ORDER BY timestamp DESC + LIMIT 1;` + const result: Prisma.AggregateScalarFieldEnum[] = await this.prisma.$queryRaw(query) + if (result.length == 1) { + return result[0] + } else { + throw Error(`Expected one row. Received ${result.length}`) + } } } diff --git a/api/src/proxy/dto/proxy.ts b/api/src/proxy/dto/proxy.ts index 20ee75684..2c37448cb 100644 --- a/api/src/proxy/dto/proxy.ts +++ b/api/src/proxy/dto/proxy.ts @@ -9,4 +9,7 @@ export class ProxyDto { @ApiProperty() port: number + + @ApiProperty() + location?: string } diff --git a/api/src/proxy/proxy.service.spec.ts b/api/src/proxy/proxy.service.spec.ts index 835cfb091..8c8afefe8 100644 --- a/api/src/proxy/proxy.service.spec.ts +++ b/api/src/proxy/proxy.service.spec.ts @@ -29,13 +29,26 @@ describe('ProxyService', () => { const proxyData = { protocol: 'http', host: '127.0.0.1', - port: 80 + port: 80, + location: 'kr' } const proxyObj = await proxy.create(proxyData) expect(proxyObj.protocol).toBe(proxyData.protocol) expect(proxyObj.host).toBe(proxyData.host) expect(proxyObj.port).toBe(proxyData.port) + expect(proxyObj.location).toBe(proxyData.location) + + const proxyDataWithoutLocation = { + protocol: 'http', + host: '127.0.0.2', + port: 80 + } + const proxyObjWithoutLocation = await proxy.create(proxyDataWithoutLocation) + expect(proxyObjWithoutLocation.protocol).toBe(proxyDataWithoutLocation.protocol) + expect(proxyObjWithoutLocation.host).toBe(proxyDataWithoutLocation.host) + expect(proxyObjWithoutLocation.port).toBe(proxyDataWithoutLocation.port) + expect(proxyObjWithoutLocation.location).toBe(null) // The same proxy cannot be defined twice await expect(async () => { await proxy.create(proxyData) diff --git a/cli/src/proxy.ts b/cli/src/proxy.ts index 2d0cd797d..61517bfc2 100644 --- a/cli/src/proxy.ts +++ b/cli/src/proxy.ts @@ -1,6 +1,6 @@ import axios from 'axios' import { command, subcommands, option, string as cmdstring, number } from 'cmd-ts' -import { idOption, buildUrl, isOraklNetworkApiHealthy } from './utils' +import { idOption, buildUrl, isOraklNetworkApiHealthy, proxyOptionalOption } from './utils' import { ORAKL_NETWORK_API_URL } from './settings' const PROXY_ENDPOINT = buildUrl(ORAKL_NETWORK_API_URL, 'proxy') @@ -30,7 +30,8 @@ export function proxySub() { port: option({ type: number, long: 'port' - }) + }), + location: proxyOptionalOption }, handler: insertHandler() }) @@ -70,16 +71,18 @@ export function insertHandler() { async function wrapper({ protocol, host, - port + port, + location }: { protocol: string host: string port: number + location?: string }) { if (!(await isOraklNetworkApiHealthy())) return try { - const response = (await axios.post(PROXY_ENDPOINT, { protocol, host, port }))?.data + const response = (await axios.post(PROXY_ENDPOINT, { protocol, host, port, location }))?.data console.dir(response, { depth: null }) } catch (e) { console.error('Proxy was not inserted. Reason:') diff --git a/cli/src/utils.ts b/cli/src/utils.ts index 02a027057..61d40f669 100644 --- a/cli/src/utils.ts +++ b/cli/src/utils.ts @@ -18,6 +18,11 @@ export const fetcherTypeOptionalOption = option({ long: 'fetcherType' }) +export const proxyOptionalOption = option({ + type: optional(cmdstring), + long: 'location' +}) + export const idOption = option({ type: cmdnumber, long: 'id' diff --git a/core/package.json b/core/package.json index 8753b1268..09bc27191 100644 --- a/core/package.json +++ b/core/package.json @@ -1,6 +1,6 @@ { "name": "@bisonai/orakl-core", - "version": "0.5.0", + "version": "0.5.1", "type": "module", "description": "The Orakl Network Core", "files": [ diff --git a/core/src/errors.ts b/core/src/errors.ts index 656fcbfb6..864f76ce1 100644 --- a/core/src/errors.ts +++ b/core/src/errors.ts @@ -61,5 +61,16 @@ export enum OraklErrorCode { CaverTxTransactionFailed, DelegatorServerIssue, FailedInsertData, - FailedInsertAggregatedData + FailedInsertAggregatedData, + AxiosBadOptionValue, + AxiosBadOption, + AxiosTimeOut, + AxiosNetworkError, + AxiosTooManyRedirects, + AxiosDeprecated, + AxiosBadResponse, + AxiosBadRequest, + AxiosCanceledByUser, + AxiosNotSupported, + AxiosInvalidUrl } diff --git a/core/src/reporter/utils.ts b/core/src/reporter/utils.ts index 9f9230266..573f57f67 100644 --- a/core/src/reporter/utils.ts +++ b/core/src/reporter/utils.ts @@ -4,9 +4,9 @@ import Caver from 'caver-js' import { ethers } from 'ethers' import { Logger } from 'pino' import { OraklError, OraklErrorCode } from '../errors' -import { ORAKL_NETWORK_DELEGATOR_URL } from '../settings' +import { DELEGATOR_TIMEOUT, ORAKL_NETWORK_DELEGATOR_URL } from '../settings' import { ITransactionData } from '../types' -import { add0x, buildUrl } from '../utils' +import { add0x, buildUrl, getOraklErrorCode } from '../utils' const FILE_NAME = import.meta.url @@ -176,15 +176,14 @@ export async function sendTransactionDelegatedFee({ const endpoint = buildUrl(ORAKL_NETWORK_DELEGATOR_URL, `sign`) let response + try { - response = ( - await axios.post(endpoint, { - ...transactionData - }) - )?.data + response = (await axios.post(endpoint, { ...transactionData }, { timeout: DELEGATOR_TIMEOUT })) + ?.data _logger.debug(response) } catch (e) { - throw new OraklError(OraklErrorCode.DelegatorServerIssue) + const errorCode = getOraklErrorCode(e, OraklErrorCode.DelegatorServerIssue) + throw new OraklError(errorCode) } try { diff --git a/core/src/settings.ts b/core/src/settings.ts index b4e1169e0..761d9fb5c 100644 --- a/core/src/settings.ts +++ b/core/src/settings.ts @@ -7,6 +7,8 @@ export const ORAKL_NETWORK_API_URL = export const ORAKL_NETWORK_DELEGATOR_URL = process.env.ORAKL_NETWORK_DELEGATOR_URL || 'http://localhost:3002/api/v1' +export const DELEGATOR_TIMEOUT = Number(process.env.DELEGATOR_TIMEOUT) || 3000 + export const DEPLOYMENT_NAME = process.env.DEPLOYMENT_NAME || 'orakl' export const NODE_ENV = process.env.NODE_ENV export const HEALTH_CHECK_PORT = process.env.HEALTH_CHECK_PORT diff --git a/core/src/utils.ts b/core/src/utils.ts index 342c6234f..95a3cbecf 100644 --- a/core/src/utils.ts +++ b/core/src/utils.ts @@ -4,6 +4,7 @@ import * as Fs from 'node:fs/promises' import os from 'node:os' import type { RedisClientType } from 'redis' import { createClient } from 'redis' +import { OraklErrorCode } from './errors' import { SLACK_WEBHOOK_URL } from './settings' export async function loadJson(filepath) { @@ -123,3 +124,32 @@ export function buildUrl(host: string, path: string) { const url = [host, path].join('/') return url.replace(/([^:]\/)\/+/g, '$1') } + +// axios errors are defined in official repo (https://github.com/axios/axios#error-types) +export const getOraklErrorCode = (e, defaultErrorCode) => { + if (e.code == 'ERR_BAD_OPTION_VALUE') { + return OraklErrorCode.AxiosBadOptionValue + } else if (e.code == 'ERR_BAD_OPTION') { + return OraklErrorCode.AxiosBadOption + } else if (e.code == 'ECONNABORTED' || e.code == 'ETIMEDOUT') { + return OraklErrorCode.AxiosTimeOut + } else if (e.code == 'ERR_NETWORK') { + return OraklErrorCode.AxiosNetworkError + } else if (e.code == 'ERR_FR_TOO_MANY_REDIRECTS') { + return OraklErrorCode.AxiosTooManyRedirects + } else if (e.code == 'ERR_DEPRECATED') { + return OraklErrorCode.AxiosDeprecated + } else if (e.code == 'ERR_BAD_RESPONSE') { + return OraklErrorCode.AxiosBadResponse + } else if (e.code == 'ERR_BAD_REQUEST') { + return OraklErrorCode.AxiosBadRequest + } else if (e.code == 'ERR_CANCELED') { + return OraklErrorCode.AxiosCanceledByUser + } else if (e.code == 'ERR_NOT_SUPPORT') { + return OraklErrorCode.AxiosNotSupported + } else if (e.code == 'ERR_INVALID_URL') { + return OraklErrorCode.AxiosInvalidUrl + } else { + return defaultErrorCode + } +} diff --git a/fetcher/src/job/job.types.ts b/fetcher/src/job/job.types.ts index cf4a717b3..7293e41f5 100644 --- a/fetcher/src/job/job.types.ts +++ b/fetcher/src/job/job.types.ts @@ -78,4 +78,5 @@ export interface IProxy { protocol: string | undefined host: string | undefined port: number | undefined + location?: string | undefined } diff --git a/fetcher/src/job/job.utils.ts b/fetcher/src/job/job.utils.ts index 3cfb7d189..941140db7 100644 --- a/fetcher/src/job/job.utils.ts +++ b/fetcher/src/job/job.utils.ts @@ -180,7 +180,18 @@ export function extractFeeds( const feeds = adapter.feeds.map((f) => { let proxy: IProxy try { - proxy = proxySelector(f.definition.url) + if (!f.definition.location) { + proxy = proxySelector(f.definition.url) + } else { + const availableProxies = proxies.filter( + (item) => item.location && item.location === f.definition.location + ) + if (availableProxies.length == 0) { + throw `no proxies available for location:${f.definition.location}` + } + const randomIndex = Math.floor(Math.random() * availableProxies.length) + proxy = availableProxies[randomIndex] + } } catch (e) { logger.error('Assigning proxy has failed') logger.error(e)