From 85634293306fca9aaa3ab5ae06a114149ccc8911 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 11:18:52 +0000 Subject: [PATCH 01/11] add datasets on ComputeAsset, new start compute fn --- src/@types/Compute.ts | 10 +++++ src/services/Provider.ts | 85 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 93 insertions(+), 2 deletions(-) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 81142c7cb..5c6638b97 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -59,7 +59,17 @@ export interface ComputeOutput { whitelist?: string[] } +export enum EncryptMethod { + AES = 'AES', + ECIES = 'ECIES' +} +export interface BaseFileObject { + type: string + encryptedBy?: string + encryptMethod?: EncryptMethod +} export interface ComputeAsset { + fileObject?: BaseFileObject // C2D v2 documentId: string serviceId: string transferTxId?: string diff --git a/src/services/Provider.ts b/src/services/Provider.ts index fd899f73b..c717ad655 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -524,7 +524,7 @@ export class Provider { return consumeUrl } - /** Instruct the provider to start a compute job + /** Instruct the provider to start a compute job (Old C2D V1) Kept for now, for backwards compatibility * @param {string} providerUri The provider URI. * @param {Signer} signer The consumer signer object. * @param {string} computeEnv The compute environment. @@ -535,7 +535,7 @@ export class Provider { * @param {ComputeOutput} output The compute job output settings. * @return {Promise} The compute job or jobs. */ - public async computeStart( + public async computeStartV1( providerUri: string, consumer: Signer, computeEnv: string, @@ -607,6 +607,87 @@ export class Provider { return null } + /** Instruct the provider to start a compute job (new C2D V2) + * @param {string} providerUri The provider URI. + * @param {Signer} signer The consumer signer object. + * @param {string} computeEnv The compute environment. + * @param {ComputeAsset} datasets The dataset to start compute on + additionalDatasets (the additional datasets if that is the case) + * @param {ComputeAlgorithm} algorithm The algorithm to start compute with. + * @param {AbortSignal} signal abort signal + * @param {ComputeOutput} output The compute job output settings. + * @return {Promise} The compute job or jobs. + */ + public async computeStart( + providerUri: string, + consumer: Signer, + computeEnv: string, + datasets: ComputeAsset[], + algorithm: ComputeAlgorithm, + signal?: AbortSignal, + output?: ComputeOutput + ): Promise { + const providerEndpoints = await this.getEndpoints(providerUri) + const serviceEndpoints = await this.getServiceEndpoints( + providerUri, + providerEndpoints + ) + const computeStartUrl = this.getEndpointURL(serviceEndpoints, 'computeStart') + ? this.getEndpointURL(serviceEndpoints, 'computeStart').urlPath + : null + + const consumerAddress = await consumer.getAddress() + const nonce = ( + (await this.getNonce( + providerUri, + consumerAddress, + signal, + providerEndpoints, + serviceEndpoints + )) + 1 + ).toString() + + let signatureMessage = consumerAddress + signatureMessage += datasets[0].documentId + signatureMessage += nonce + const signature = await this.signProviderRequest(consumer, signatureMessage) + const payload = Object() + payload.consumerAddress = consumerAddress + payload.signature = signature + payload.nonce = nonce + payload.environment = computeEnv + payload.datasets = datasets + payload.algorithm = algorithm + + if (output) payload.output = output + if (!computeStartUrl) return null + let response + try { + response = await fetch(computeStartUrl, { + method: 'POST', + body: JSON.stringify(payload), + headers: { 'Content-Type': 'application/json' }, + signal + }) + } catch (e) { + LoggerInstance.error('Compute start failed:') + LoggerInstance.error(e) + LoggerInstance.error('Payload was:', payload) + throw new Error('HTTP request failed calling Provider') + } + if (response?.ok) { + const params = await response.json() + return params + } + LoggerInstance.error( + 'Compute start failed: ', + response.status, + response.statusText, + await response.json() + ) + LoggerInstance.error('Payload was:', payload) + return null + } + /** Instruct the provider to Stop the execution of a to stop a compute job. * @param {string} did the asset did * @param {string} consumerAddress The consumer address. From 29eb7fbfdc7262aad3656cb54d6252ca7db88306 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 18:01:50 +0000 Subject: [PATCH 02/11] fix type mismatch new c2d v2 --- test/integration/ComputeExamples.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration/ComputeExamples.test.ts b/test/integration/ComputeExamples.test.ts index da931e92c..9779fb7aa 100644 --- a/test/integration/ComputeExamples.test.ts +++ b/test/integration/ComputeExamples.test.ts @@ -655,7 +655,7 @@ describe('Compute-to-data example tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) From a6ad053cd068d0d1cdb82893102e9c7f322e2930 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 18:26:43 +0000 Subject: [PATCH 03/11] fix tests --- ComputeExamples.md | 2 +- test/integration/ComputeFlow.test.ts | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/ComputeExamples.md b/ComputeExamples.md index 8440582f5..f3af29848 100644 --- a/ComputeExamples.md +++ b/ComputeExamples.md @@ -655,7 +655,7 @@ Let's have 5 minute of compute access providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) diff --git a/test/integration/ComputeFlow.test.ts b/test/integration/ComputeFlow.test.ts index 31558c986..24c3dd9be 100644 --- a/test/integration/ComputeFlow.test.ts +++ b/test/integration/ComputeFlow.test.ts @@ -461,7 +461,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) freeEnvDatasetTxId = assets[0].transferTxId @@ -530,7 +530,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) assert(computeJobs, 'Cannot start compute job') @@ -594,7 +594,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) paidEnvDatasetTxId = assets[0].transferTxId @@ -670,7 +670,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) assert(computeJobs, 'Cannot start compute job') @@ -762,7 +762,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) // freeEnvDatasetTxId = assets[0].transferTxId @@ -846,7 +846,7 @@ describe('Compute flow tests', async () => { providerUrl, consumerAccount, computeEnv.id, - assets[0], + assets, algo ) // freeEnvDatasetTxId = assets[0].transferTxId From e1c5194f43259d957c386454b6e7c890b421d48f Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 31 Oct 2024 19:33:54 +0000 Subject: [PATCH 04/11] fix tests, keep backwards compatibility due to tests running with existing provider --- src/services/Provider.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/services/Provider.ts b/src/services/Provider.ts index c717ad655..2de6a0638 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -626,6 +626,7 @@ export class Provider { signal?: AbortSignal, output?: ComputeOutput ): Promise { + console.log('called new compute start method...') const providerEndpoints = await this.getEndpoints(providerUri) const serviceEndpoints = await this.getServiceEndpoints( providerUri, @@ -655,9 +656,12 @@ export class Provider { payload.signature = signature payload.nonce = nonce payload.environment = computeEnv + // kept for backwards compatibility (tests running against existing provider) + payload.dataset = datasets[0] + // new field for C2D v2 payload.datasets = datasets payload.algorithm = algorithm - + // if (additionalDatasets) payload.additionalDatasets = additionalDatasets if (output) payload.output = output if (!computeStartUrl) return null let response From 59daaae3d1fdcb3bc646e4c161292d96d5e1a2a7 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Mon, 4 Nov 2024 17:44:45 +0000 Subject: [PATCH 05/11] support for starting a free env compute job --- src/@types/Compute.ts | 1 + src/services/Provider.ts | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 5c6638b97..83df541bf 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -22,6 +22,7 @@ export interface ComputeEnvironment { storageExpiry: number maxJobDuration: number lastSeen: number + free: boolean } export interface ComputeResult { diff --git a/src/services/Provider.ts b/src/services/Provider.ts index 2de6a0638..a395ac570 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -615,6 +615,7 @@ export class Provider { * @param {ComputeAlgorithm} algorithm The algorithm to start compute with. * @param {AbortSignal} signal abort signal * @param {ComputeOutput} output The compute job output settings. + * @param {boolean} freeEnvironment is it a free environment? uses different route * @return {Promise} The compute job or jobs. */ public async computeStart( @@ -624,7 +625,8 @@ export class Provider { datasets: ComputeAsset[], algorithm: ComputeAlgorithm, signal?: AbortSignal, - output?: ComputeOutput + output?: ComputeOutput, + freeEnvironment?: boolean ): Promise { console.log('called new compute start method...') const providerEndpoints = await this.getEndpoints(providerUri) @@ -632,9 +634,17 @@ export class Provider { providerUri, providerEndpoints ) - const computeStartUrl = this.getEndpointURL(serviceEndpoints, 'computeStart') - ? this.getEndpointURL(serviceEndpoints, 'computeStart').urlPath - : null + let computeStartUrl = null + + if (freeEnvironment) { + computeStartUrl = this.getEndpointURL(serviceEndpoints, 'freeCompute') + ? this.getEndpointURL(serviceEndpoints, 'freeCompute').urlPath + : null + } else { + computeStartUrl = this.getEndpointURL(serviceEndpoints, 'computeStart') + ? this.getEndpointURL(serviceEndpoints, 'computeStart').urlPath + : null + } const consumerAddress = await consumer.getAddress() const nonce = ( From 66b8da15d0fadcdd69c52513e9b4f04cefc1b146 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 5 Nov 2024 16:54:14 +0000 Subject: [PATCH 06/11] minor fix --- src/@types/Compute.ts | 7 +++++++ test/integration/helpers.ts | 3 ++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 83df541bf..26a6dbd07 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -60,6 +60,12 @@ export interface ComputeOutput { whitelist?: string[] } +export enum FileObjectType { + URL = 'url', + IPFS = 'ipfs', + ARWEAVE = 'arweave' +} + export enum EncryptMethod { AES = 'AES', ECIES = 'ECIES' @@ -78,6 +84,7 @@ export interface ComputeAsset { } export interface ComputeAlgorithm { + fileObject?: BaseFileObject // C2D v2 documentId?: string serviceId?: string meta?: MetadataAlgorithm diff --git a/test/integration/helpers.ts b/test/integration/helpers.ts index bafe4dea2..efafd5848 100644 --- a/test/integration/helpers.ts +++ b/test/integration/helpers.ts @@ -133,7 +133,8 @@ export async function handleComputeOrder( - have validOrder and providerFees -> then order is valid but providerFees are not valid, we need to call reuseOrder and pay only providerFees - no validOrder -> we need to call startOrder, to pay 1 DT & providerFees */ - if (order.providerFee && order.providerFee.providerFeeAmount) { + const hasProviderFees = order.providerFee && order.providerFee.providerFeeAmount + if (hasProviderFees && Number(order.providerFee.providerFeeAmount) > 0) { await approveWei( payerAccount, config, From cd936c24a989633d3d8e71b908a375802d2e2970 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Wed, 6 Nov 2024 09:44:30 +0000 Subject: [PATCH 07/11] add file object types --- src/@types/Compute.ts | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/@types/Compute.ts b/src/@types/Compute.ts index 26a6dbd07..85b5b123b 100644 --- a/src/@types/Compute.ts +++ b/src/@types/Compute.ts @@ -70,11 +70,30 @@ export enum EncryptMethod { AES = 'AES', ECIES = 'ECIES' } + +export interface HeadersObject { + [key: string]: string +} + export interface BaseFileObject { type: string encryptedBy?: string encryptMethod?: EncryptMethod } + +export interface UrlFileObject extends BaseFileObject { + url: string + method: string + headers?: [HeadersObject] +} + +export interface IpfsFileObject extends BaseFileObject { + hash: string +} + +export interface ArweaveFileObject extends BaseFileObject { + transactionId: string +} export interface ComputeAsset { fileObject?: BaseFileObject // C2D v2 documentId: string From 098c29fc056b2a5c40643ed911f01a17f6a73c3a Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Thu, 7 Nov 2024 15:45:44 +0000 Subject: [PATCH 08/11] enforce asset files object type --- src/utils/Assets.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utils/Assets.ts b/src/utils/Assets.ts index bd1aa7b9a..9916c7a47 100644 --- a/src/utils/Assets.ts +++ b/src/utils/Assets.ts @@ -21,6 +21,7 @@ import { ProviderInstance } from '../services/Provider' import ERC20Template from '@oceanprotocol/contracts/artifacts/contracts/interfaces/IERC20Template.sol/IERC20Template.json' import AccessListFactory from '@oceanprotocol/contracts/artifacts/contracts/accesslists/AccessListFactory.sol/AccessListFactory.json' import ERC20Template4 from '@oceanprotocol/contracts/artifacts/contracts/templates/ERC20Template4.sol/ERC20Template4.json' +import { FileObjectType } from '../@types' // import * as hre from 'hardhat' @@ -202,6 +203,15 @@ export async function createAsset( mpFeeAddress: ZERO_ADDRESS } + if ( + !assetUrl.type || + ![FileObjectType.ARWEAVE, FileObjectType.IPFS, FileObjectType.URL].includes( + assetUrl.type.toLowerCase() + ) + ) { + console.log('Missing or invalid files object type, defaulting to "url"') + assetUrl.type = FileObjectType.URL + } // include fileObject in the DT constructor if (config.sdk === 'oasis') { datatokenParams.filesObject = assetUrl From 71c59230db99a08a6c28df66c26e32ca61c71089 Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 12 Nov 2024 12:18:32 +0000 Subject: [PATCH 09/11] support for consumer signature on initialize compute --- src/services/Provider.ts | 88 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 87 insertions(+), 1 deletion(-) diff --git a/src/services/Provider.ts b/src/services/Provider.ts index a395ac570..d7c987a4a 100644 --- a/src/services/Provider.ts +++ b/src/services/Provider.ts @@ -418,7 +418,7 @@ export class Provider { * @param {AbortSignal} signal abort signal * @return {Promise} ProviderComputeInitialize data */ - public async initializeCompute( + public async initializeComputeV1( assets: ComputeAsset[], algorithm: ComputeAlgorithm, computeEnv: string, @@ -470,6 +470,90 @@ export class Provider { throw new Error(JSON.stringify(resolvedResponse)) } + /** Initializes the provider for a compute request. + * @param {ComputeAsset[]} assets The datasets array to initialize compute request. + * @param {ComputeAlgorithmber} algorithm The algorithm to use. + * @param {string} computeEnv The compute environment. + * @param {number} validUntil The job expiration date. + * @param {string} providerUri The provider URI. + * @param {Signer} signer caller address + * @param {AbortSignal} signal abort signal + * @return {Promise} ProviderComputeInitialize data + */ + public async initializeCompute( + assets: ComputeAsset[], + algorithm: ComputeAlgorithm, + computeEnv: string, + validUntil: number, + providerUri: string, + signer: Signer, + signal?: AbortSignal + ): Promise { + const providerEndpoints = await this.getEndpoints(providerUri) + const serviceEndpoints = await this.getServiceEndpoints( + providerUri, + providerEndpoints + ) + + // Diff from V1. We might need a signature to get the files object, specially if dealing with confidential evm and template 4 + // otherwise it can be ignored + const consumerAddress = await signer.getAddress() + const nonce = ( + (await this.getNonce( + providerUri, + consumerAddress, + signal, + providerEndpoints, + serviceEndpoints + )) + 1 + ).toString() + + // same signed message as for start compute (consumer address + did[0] + nonce) + let signatureMessage = consumerAddress + signatureMessage += assets[0].documentId + signatureMessage += nonce + const signature = await this.signProviderRequest(signer, signatureMessage) + + const providerData = { + datasets: assets, + algorithm, + compute: { env: computeEnv, validUntil }, + consumerAddress, + signature + } + const initializeUrl = this.getEndpointURL(serviceEndpoints, 'initializeCompute') + ? this.getEndpointURL(serviceEndpoints, 'initializeCompute').urlPath + : null + if (!initializeUrl) return null + + let response + try { + response = await fetch(initializeUrl, { + method: 'POST', + body: JSON.stringify(providerData), + headers: { 'Content-Type': 'application/json' }, + signal + }) + } catch (e) { + LoggerInstance.error('Initialize compute failed: ') + LoggerInstance.error(e) + throw new Error('ComputeJob cannot be initialized') + } + if (response?.ok) { + const params = await response.json() + return params + } + const resolvedResponse = await response.json() + LoggerInstance.error( + 'Initialize compute failed: ', + response.status, + response.statusText, + resolvedResponse + ) + LoggerInstance.error('Payload was:', providerData) + throw new Error(JSON.stringify(resolvedResponse)) + } + /** * Gets the download URL. * @param {string} did - The DID. @@ -629,6 +713,8 @@ export class Provider { freeEnvironment?: boolean ): Promise { console.log('called new compute start method...') + console.log('datasets: ', datasets) + console.log('algorithm: ', algorithm) const providerEndpoints = await this.getEndpoints(providerUri) const serviceEndpoints = await this.getServiceEndpoints( providerUri, From 81a60a9e514adeee795bd2b1fc4a503669ffdc3c Mon Sep 17 00:00:00 2001 From: paulo-ocean Date: Tue, 12 Nov 2024 12:40:05 +0000 Subject: [PATCH 10/11] update docs and tests, pass signer account, not signer address --- ComputeExamples.md | 2 +- docs/classes/Provider.md | 2 +- test/integration/ComputeExamples.test.ts | 2 +- test/integration/ComputeFlow.test.ts | 12 ++++++------ 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ComputeExamples.md b/ComputeExamples.md index bd9b7ee7d..83dc646f9 100644 --- a/ComputeExamples.md +++ b/ComputeExamples.md @@ -627,7 +627,7 @@ Let's have 5 minute of compute access computeEnv.id, computeValidUntil, providerUrl, - await consumerAccount.getAddress() + consumerAccount ) ```