Skip to content

Commit

Permalink
chore: merge hexArray job support and caching fix into releases/v1.1.0 (
Browse files Browse the repository at this point in the history 
#1177)

* fix: conversion of hex to decimal (#1167)

* fix: conversion of hex to decimal

* fix: convertToNumber tests fixed

* feat: added support for job returning results in hex array (#1174)

* feat: added support for jobs returning result in hex array

* feat: added tests for hex array decoding and regex functions

* refactor: fixed golangci-lint errors

* refactor: used hash of url and body as a cache key (#1176)

* refactor: used hash of url and body as cache key instead of just url

* fix: replaced with new cacheKey while reading from cache

* refactor: added tests for different job cases

* refcator: added missing coverage test cases

* refcator: fixed tests

* refactor: fixted tests by replcing binance API with coinbase API

* refactor: job index changed in test name

* fix: conversion of hex to decimal (#1167) (#1169) (#1178)

* fix: conversion of hex to decimal

* fix: convertToNumber tests fixed
  • Loading branch information
@Yashk767
Yashk767 authored Jan 17, 2024
1 parent 703aec5 commit c01811b
Show file tree
Hide file tree
Showing 8 changed files with 549 additions and 65 deletions.
6 changes: 6 additions & 0 deletions core/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,3 +66,9 @@ var SwitchClientDuration = 5 * EpochLength

// HexReturnType is the ReturnType for a job if that job returns a hex value
var HexReturnType = "hex"

// HexArrayReturnType is the ReturnType for a job if that job returns a hex array value
var HexArrayReturnType = "^hexArray\\[\\d+\\]$"

// HexArrayExtractIndexRegex will be used as a regular expression to extract index from hexArray return type
var HexArrayExtractIndexRegex = `^hexArray\[(\d+)\]$`
10 changes: 8 additions & 2 deletions utils/api.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,13 @@ func GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, localCache *cache.L
Timeout: time.Duration(HTTPTimeout) * time.Second,
}

cachedData, found := localCache.Read(dataSourceURLStruct.URL)
cacheKey, err := generateCacheKey(dataSourceURLStruct.URL, dataSourceURLStruct.Body)
if err != nil {
log.Errorf("Error in generating cache key for API %s: %v", dataSourceURLStruct.URL, err)
return nil, err
}

cachedData, found := localCache.Read(cacheKey)
if found {
log.Debugf("Getting Data for URL %s from local cache...", dataSourceURLStruct.URL)
return cachedData, nil
Expand All @@ -36,7 +42,7 @@ func GetDataFromAPI(dataSourceURLStruct types.DataSourceURL, localCache *cache.L
}

// Storing the data into cache
localCache.Update(response, dataSourceURLStruct.URL, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())
localCache.Update(response, cacheKey, time.Now().Add(time.Second*time.Duration(core.StateLength)).Unix())
return response, nil
}

Expand Down
13 changes: 13 additions & 0 deletions utils/api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,6 +150,19 @@ func TestGetDataFromAPI(t *testing.T) {
want: nil,
wantErr: true,
},
{
name: "Gneerating cache key throws error",
args: args{
urlStruct: types.DataSourceURL{
URL: "http://example.com",
Body: map[string]interface{}{
"key": func() {}, // functions cannot be marshaled and will cause an error
},
},
},
want: nil,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
139 changes: 76 additions & 63 deletions utils/asset_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -554,95 +554,95 @@ func TestGetAllCollections(t *testing.T) {

func TestGetDataToCommitFromJobs(t *testing.T) {
jobsArray := []bindings.StructsJob{
{Id: 1, SelectorType: 1, Weight: 100,
Power: 2, Name: "ethusd_gemini", Selector: "last",
Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`,
}, {Id: 2, SelectorType: 1, Weight: 100,
{Id: 1, SelectorType: 0, Weight: 10,
Power: 2, Name: "ethusd_gemini", Selector: "last",
Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`,
Url: "https://api.gemini.com/v1/pubticker/ethusd",
},
{Id: 2, SelectorType: 0, Weight: 20,
Power: 2, Name: "ethusd_kraken", Selector: "result.XETHZUSD.c[0]",
Url: `{"type": "GET","url": "https://api.kraken.com/0/public/Ticker?pair=ETHUSD","body": {},"header": {}}`,
},
{Id: 3, SelectorType: 0, Weight: 30,
Power: 2, Name: "ethusd_kucoin", Selector: "data.ETH",
Url: `{"type": "GET","url": "https://api.kucoin.com/api/v1/prices?base=USD&currencies=ETH","body": {},"header": {}}`,
},
{Id: 4, SelectorType: 0, Weight: 40,
Power: 2, Name: "ethusd_coinbase", Selector: "data.amount",
Url: `{"type": "GET","url": "https://api.coinbase.com/v2/prices/ETH-USD/spot","body": {},"header": {}}`,
},
// This job returns an error which will not add any value to data or weight array
{Id: 5, SelectorType: 0, Weight: 10,
Power: 2, Name: "ethusd_gemini_incorrect", Selector: "last1",
Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd1","body": {},"header": {}}`,
},
{Id: 6, SelectorType: 0, Weight: 100,
Power: 6, Name: "ethusd_uniswapv2", Selector: "result",
Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","id":7269270904970082,"method":"eth_call","params":[{"from":"0x0000000000000000000000000000000000000000","data":"0xd06ca61f0000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000200000000000000000000000050de6856358cc35f3a9a57eaaa34bd4cb707d2cd0000000000000000000000008e870d67f660d95d5be530380d0ec0bd388289e1","to":"0x7a250d5630b4cf539739df2c5dacb4c659f2488d"},"latest"]},"header": {"content-type": "application/json"}, "returnType": "hexArray[1]"}`,
},
{Id: 7, SelectorType: 0, Weight: 100,
Power: 2, Name: "ethusd_uniswapv3", Selector: "result",
Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`,
},
// This is a duplicate job to check if cache is working correctly for POST Jobs
{Id: 8, SelectorType: 0, Weight: 100,
Power: 2, Name: "ethusd_uniswapv3_duplicate", Selector: "result",
Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`,
},
// This is a duplicate job to check if cache is working correctly for GET Jobs
{Id: 9, SelectorType: 0, Weight: 10,
Power: 2, Name: "ethusd_gemini_duplicate", Selector: "last",
Url: "https://api.gemini.com/v1/pubticker/ethusd",
},
}

type args struct {
jobPath string
jobPathErr error
overrideJobData map[string]*types.StructsJob
overrideJobDataErr error
dataToAppend *big.Int
dataToAppendErr error
jobs []bindings.StructsJob
}

tests := []struct {
name string
args args
want []*big.Int
wantErr bool
name string
args args
wantArrayLength int
wantErr bool
}{
{
name: "Test 1: When GetDataToCommitFromJobs() executes successfully",
name: "Test 1: Getting values from set of jobs of length 4",
args: args{
jobPath: "",
overrideJobData: map[string]*types.StructsJob{"1": {
Id: 2, SelectorType: 1, Weight: 100,
Power: 2, Name: "ethusd_gemini", Selector: "last",
Url: `{"type": "GET","url": "https://api.gemini.com/v1/pubticker/ethusd","body": {},"header": {}}`,
}},
dataToAppend: big.NewInt(1),
jobs: jobsArray[:4],
},
want: []*big.Int{big.NewInt(1), big.NewInt(1)},
wantErr: false,
wantArrayLength: 4,
},
{
name: "Test 2: When there is an error in getting overrideJobData",
name: "Test 2: Getting values from set of jobs of length 2",
args: args{
jobPath: "",
overrideJobDataErr: errors.New("overrideJobData error"),
dataToAppend: big.NewInt(1),
jobs: jobsArray[:2],
},
want: []*big.Int{big.NewInt(1), big.NewInt(1)},
wantErr: false,
wantArrayLength: 2,
},
{
name: "Test 3: When there is an error in getting jobPath",
name: "Test 3: Getting values from whole set of jobs of length 9 but job at index 5 reports an error",
args: args{
jobPathErr: errors.New("jobPath error"),
overrideJobData: map[string]*types.StructsJob{},
dataToAppend: big.NewInt(1),
jobs: jobsArray,
},
want: []*big.Int{big.NewInt(1), big.NewInt(1)},
wantErr: false,
},
{
name: "Test 4: When there is an error in getting dataToAppend",
args: args{
jobPath: "",
overrideJobData: map[string]*types.StructsJob{},
dataToAppendErr: errors.New("dataToAppend error"),
},
want: nil,
wantErr: false,
wantArrayLength: 8,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
utilsMock := new(mocks.Utils)
pathMock := new(mocks.PathUtils)
UtilsInterface = &UtilsStruct{}
lc := cache.NewLocalCache(time.Second * 20)

optionsPackageStruct := OptionsPackageStruct{
UtilsInterface: utilsMock,
PathInterface: pathMock,
}
utils := StartRazor(optionsPackageStruct)

utilsMock.On("GetDataToCommitFromJob", mock.Anything, mock.Anything).Return(tt.args.dataToAppend, tt.args.dataToAppendErr)

got, _, err := utils.GetDataToCommitFromJobs(jobsArray, &cache.LocalCache{})
gotDataArray, gotWeightArray, err := UtilsInterface.GetDataToCommitFromJobs(tt.args.jobs, lc)
if (err != nil) != tt.wantErr {
t.Errorf("GetDataToCommitFromJobs() error = %v, wantErr %v", err, tt.wantErr)
t.Errorf("GetDataToCommitFromJob() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("GetDataToCommitFromJobs() got = %v, want %v", got, tt.want)

if len(gotDataArray) != tt.wantArrayLength || len(gotWeightArray) != tt.wantArrayLength {
t.Errorf("GetDataToCommitFromJobs() got = %v, want %v", gotDataArray, tt.wantArrayLength)
}
fmt.Println("Got Data Array: ", gotDataArray)
fmt.Println("Got WeightArray: ", gotWeightArray)
})
}
}
Expand All @@ -658,11 +658,16 @@ func TestGetDataToCommitFromJob(t *testing.T) {
Url: "https://api.gemini.com/v1/pubticker/ethusd/apiKey=${SAMPLE_API_KEY_NEW}",
}

postJob := bindings.StructsJob{Id: 1, SelectorType: 0, Weight: 100,
postJobUniswapV3 := bindings.StructsJob{Id: 1, SelectorType: 0, Weight: 100,
Power: 2, Name: "ethusd_sample", Selector: "result",
Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","method":"eth_call","params":[{"to":"0xb27308f9f90d607463bb33ea1bebb41c27ce5ab6","data":"0xf7729d43000000000000000000000000c02aaa39b223fe8d0a0e5c4f27ead9083c756cc2000000000000000000000000a0b86991c6218b36c1d19d4a2e9eb0ce3606eb480000000000000000000000000000000000000000000000000000000000000bb80000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000000"},"latest"],"id":5},"header": {"content-type": "application/json"}, "returnType": "hex"}`,
}

postJobUniswapV2 := bindings.StructsJob{Id: 1, SelectorType: 0, Weight: 100,
Power: 6, Name: "ethusd_sample", Selector: "result",
Url: `{"type": "POST","url": "https://rpc.ankr.com/eth","body": {"jsonrpc":"2.0","id":7269270904970082,"method":"eth_call","params":[{"from":"0x0000000000000000000000000000000000000000","data":"0xd06ca61f0000000000000000000000000000000000000000000000000de0b6b3a76400000000000000000000000000000000000000000000000000000000000000000040000000000000000000000000000000000000000000000000000000000000000200000000000000000000000050de6856358cc35f3a9a57eaaa34bd4cb707d2cd0000000000000000000000008e870d67f660d95d5be530380d0ec0bd388289e1","to":"0x7a250d5630b4cf539739df2c5dacb4c659f2488d"},"latest"]},"header": {"content-type": "application/json"}, "returnType": "hexArray[1]"}`,
}

invalidDataSourceStructJob := bindings.StructsJob{Id: 1, SelectorType: 0, Weight: 100,
Power: 2, Name: "ethusd_sample", Selector: "result",
Url: `{"type": true,"url1": {}}`,
Expand Down Expand Up @@ -700,7 +705,7 @@ func TestGetDataToCommitFromJob(t *testing.T) {
{
name: "Test 3: When GetDataToCommitFromJob() executes successfully for a POST Job",
args: args{
job: postJob,
job: postJobUniswapV3,
},
wantErr: false,
},
Expand All @@ -720,6 +725,14 @@ func TestGetDataToCommitFromJob(t *testing.T) {
want: nil,
wantErr: true,
},
{
name: "Test 6: When GetDataToCommitFromJob() executes successfully for a POST uniswap v2 Job",
args: args{
job: postJobUniswapV2,
},
want: nil,
wantErr: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down
18 changes: 18 additions & 0 deletions utils/hash.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
package utils

import (
"encoding/json"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/crypto"
solsha3 "github.com/miguelmota/go-solidity-sha3"
)

func EcRecover(data, sig hexutil.Bytes) (common.Address, error) {
Expand All @@ -23,3 +25,19 @@ func SignHash(data []byte) []byte {
msg := fmt.Sprintf("\x19Ethereum Signed Message:\n%d%s", len(data), data)
return crypto.Keccak256([]byte(msg))
}

func generateCacheKey(url string, body map[string]interface{}) (string, error) {
var bodyString string
if body != nil {
// Convert the body to a JSON string
bodyBytes, err := json.Marshal(body)
if err != nil {
log.Error("Error in marshalling body: ", err)
return "", err
}
bodyString = string(bodyBytes)
}

hash := solsha3.SoliditySHA3([]string{"string", "string"}, []interface{}{url, bodyString})
return common.BytesToHash(hash).Hex(), nil
}
64 changes: 64 additions & 0 deletions utils/hash_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,67 @@ func TestEcRecover(t *testing.T) {
})
}
}

func Test_generateCacheKey(t *testing.T) {
type args struct {
url string
body map[string]interface{}
}
tests := []struct {
name string
args args
want string
wantErr bool
}{
{
name: "Test when the request is of type GET with body as nil",
args: args{
url: "https:api.gemini.com/v1/pubticker",
body: nil,
},
want: "0x9e9d7684772b773287cfd315aea97e574b7059bf63599ad1e008b9f695ab46e2",
wantErr: false,
},
{
name: "Test when the request is of type POST with body",
args: args{
url: "https://staging-v3.skalenodes.com/v1/staging-aware-chief-gianfar",
body: map[string]interface{}{"jsonrpc": "2.0", "method": "eth_chainId", "params": nil, "id": 0},
},
want: "0xecd57ac05ee0584932ac9b969f6e8a851b7a5f1bbd46ae756f48ad9e2747a0ff",
wantErr: false,
},
{
name: "Test when url and body is nil",
args: args{
url: "",
body: nil,
},
want: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
wantErr: false,
},
{
name: "Test when marshalling fails",
args: args{
url: "http://example.com",
body: map[string]interface{}{
"key": func() {}, // functions cannot be marshaled and will cause an error
},
},
want: "",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := generateCacheKey(tt.args.url, tt.args.body)
if (err != nil) != tt.wantErr {
t.Errorf("generateCacheKey() error = %v, wantErr %v", err, tt.wantErr)
return
}
if got != tt.want {
t.Errorf("generateCacheKey() got = %v, want %v", got, tt.want)
}
})
}
}
Loading

0 comments on commit c01811b

Please sign in to comment.