Feat: sort out inference code after refactor for finetune

inference.ts

@@ -0,0 +1,118 @@
+import {ethers} from 'ethers'
+import {createZGComputeNetworkBroker} from './src.ts'
+import OpenAI from 'openai'
+
+async function main() {
+    const provider = new ethers.JsonRpcProvider('http://127.0.0.1:8545')
+
+    // Step 1: Create a wallet with a private key
+    const privateKey =
+        '59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d'
+    const wallet = new ethers.Wallet(privateKey, provider)
+
+    // Step 2: Initialize the broker
+    try {
+        const broker = await createZGComputeNetworkBroker(
+            wallet,
+            '',
+            '0x0165878A594ca255338adfa4d48449f69242Eb8F',
+            ''
+        )
+
+        // List available services
+        console.log('Listing available services...')
+        const services = await broker.inference.listService()
+        services.forEach((service: any) => {
+            console.log(
+                `Service: ${service.name}, Provider: ${service.provider}, Type: ${service.serviceType}, Model: ${service.model}, URL: ${service.url}, verifiability: ${service.verifiability}`
+            )
+        })
+
+        // Select a service
+        const service = await broker.inference.getService("test")
+
+        const providerAddress = service.provider;
+
+        // create ledger
+        await broker.ledger.addLedger(0.1)
+        console.log('Creating ledger successfully.')
+
+        // deposit fund
+        const depositAmount = 0.01
+        await broker.ledger.depositFund(depositAmount)
+        console.log('Depositing funds...')
+
+        // get ledger
+        const ledger = await broker.ledger.getLedger()
+        console.log('Get ledger with fund ', ledger.totalBalance)
+
+        // Step 5: Use the Provider's Services
+        console.log('Processing a request...')
+        const serviceName = service.name
+        const content = 'hello world'
+
+        await broker.inference.settleFee(
+            providerAddress,
+            serviceName,
+            0.0000000008
+        )
+
+        // Step 5.1: Get the request metadata
+        const {endpoint, model} =
+            await broker.inference.getServiceMetadata(
+                providerAddress,
+                serviceName
+            )
+
+        // Get the request headers
+        const headers = await broker.inference.getRequestHeaders(
+            providerAddress,
+            serviceName,
+            content
+        )
+
+        // Send a request to the service
+        const openai = new OpenAI({
+            baseURL: endpoint,
+            apiKey: '',
+        })
+        const completion = await openai.chat.completions.create(
+            {
+                messages: [{role: 'system', content}],
+                model: model,
+                // @ts-expect-error guided_json is not yet public
+                guided_json: jsonSchema,
+            },
+            {
+                headers: {
+                    ...headers,
+                },
+            }
+        )
+
+        const receivedContent = completion.choices[0].message.content
+        const chatID = completion.id
+        if (!receivedContent) {
+            throw new Error('No content received.')
+        }
+        console.log('Response:', receivedContent)
+
+        // Step 7: Process the response
+        console.log('Processing a response...')
+        const isValid = await broker.inference.processResponse(
+            providerAddress,
+            serviceName,
+            receivedContent,
+            chatID
+        )
+        console.log(`Response validity: ${isValid ? 'Valid' : 'Invalid'}`)
+
+        // close service
+        await broker.inference.closeService(providerAddress)
+        console.log("Closing the service...")
+    } catch (error) {
+        console.error('Error during execution:', error)
+    }
+}
+
+main()

src.ts/broker.ts

@@ -43,7 +43,7 @@ export async function createZGComputeNetworkBroker(
     try {
         const ledger = await createLedgerBroker(signer, ledgerCA)
         // TODO: Adapts the usage of the ledger broker to initialize the inference broker.
-        const inferenceBroker = await createInferenceBroker(signer, inferenceCA)
+        const inferenceBroker = await createInferenceBroker(signer, inferenceCA, ledger)
         const fineTuningBroker = await createFineTuningBroker(
             signer,
             fineTuningCA,

src.ts/inference/broker/base.ts

@@ -172,7 +172,7 @@ export abstract class ZGServingUserBrokerBase {
         }
     }
 
-    private async calculateInputFees(extractor: Extractor, content: string) {
+    protected async calculateInputFees(extractor: Extractor, content: string) {
         const svc = await extractor.getSvcInfo()
         const inputCount = await extractor.getInputCount(content)
         const inputFee = BigInt(inputCount) * svc.inputPrice

src.ts/inference/broker/broker.ts

@@ -1,12 +1,13 @@
-import { AccountStructOutput, InferenceServingContract } from '../contract'
-import { JsonRpcSigner, Wallet } from 'ethers'
-import { RequestProcessor } from './request'
-import { ResponseProcessor } from './response'
-import { Verifier } from './verifier'
-import { AccountProcessor } from './account'
-import { ModelProcessor } from './model'
-import { Metadata } from '../../common/storage'
-import { Cache } from '../storage'
+import {AccountStructOutput, InferenceServingContract, ServiceStructOutput} from '../contract'
+import {AddressLike, JsonRpcSigner, toNumber, Wallet} from 'ethers'
+import {RequestProcessor} from './request'
+import {ResponseProcessor} from './response'
+import {Verifier} from './verifier'
+import {AccountProcessor} from './account'
+import {ModelProcessor} from './model'
+import {Metadata} from '../../common/storage'
+import {Cache} from '../storage'
+import {LedgerBroker} from '../../ledger'
 
 export class InferenceBroker {
     public requestProcessor!: RequestProcessor
@@ -17,10 +18,12 @@ export class InferenceBroker {
 
     private signer: JsonRpcSigner | Wallet
     private contractAddress: string
+    private ledger: LedgerBroker
 
-    constructor(signer: JsonRpcSigner | Wallet, contractAddress: string) {
+    constructor(signer: JsonRpcSigner | Wallet, contractAddress: string, ledger: LedgerBroker) {
         this.signer = signer
         this.contractAddress = contractAddress
+        this.ledger = ledger
     }
 
     async initialize() {
@@ -62,6 +65,22 @@ export class InferenceBroker {
         }
     }
 
+    public getService = async (svcName: string): Promise<ServiceStructOutput> => {
+        try {
+            const services = await this.listService()
+            const service = services.find(
+                (service: any) => service.name === svcName
+            )
+            if (!service) {
+                console.error('Service not found.')
+                return
+            }
+            return service
+        } catch (error) {
+            throw error
+        }
+    }
+
     /**
      * Adds a new account to the contract.
      *
@@ -148,6 +167,27 @@ export class InferenceBroker {
         }
     }
 
+    /**
+     * Check if the inference account has enough fund, if not, then transfer the difference.
+     * @param amount
+     * @param provider
+     */
+    private transferFundIfNeeded = async (amount: bigint, provider: string) => {
+        const ledger_out = await this.ledger.getLedger()
+        // available amount in ledger
+        const avail_amount = ledger_out.availableBalance
+        const diff = amount - avail_amount
+        if (diff > 0) {
+            // not enough money, transfer more from ledger
+            try {
+                // todo: do we need to check the total amount here? or just let it fail if not enough
+                this.ledger.transferFund(provider, "inference", Number(diff))
+            } catch (error) {
+                throw error
+            }
+        }
+    }
+
     /**
      * getRequestHeaders generates billing-related headers for the request
      * when the user uses the provider service.
@@ -198,6 +238,11 @@ export class InferenceBroker {
         content: string
     ) => {
         try {
+            // transfer fund from ledger to account if needed
+            await this.addAccount(providerAddress, 0)
+            const amount = await this.requestProcessor.calculateInputFee(providerAddress, svcName, content)
+            await this.transferFundIfNeeded(amount, providerAddress)
+
             return await this.requestProcessor.getRequestHeaders(
                 providerAddress,
                 svcName,
@@ -235,6 +280,11 @@ export class InferenceBroker {
         chatID?: string
     ): Promise<boolean | null> => {
         try {
+            // transfer fund from ledger to account if needed
+            await this.addAccount(providerAddress, 0)
+            const amount = await this.responseProcessor.calculateOutputFees(providerAddress, svcName, content)
+            await this.transferFundIfNeeded(amount, providerAddress)
+
             return await this.responseProcessor.processResponse(
                 providerAddress,
                 svcName,
@@ -356,6 +406,18 @@ export class InferenceBroker {
             throw error
         }
     }
+
+    /**
+     * retrive fund from inference account back to ledger
+     * @param provider
+     */
+    public closeService = async (provider: AddressLike) => {
+        try {
+            await this.ledger.retrieveFund([provider], "inference")
+        } catch (error) {
+            throw error
+        }
+    }
 }
 
 /**
@@ -370,9 +432,10 @@ export class InferenceBroker {
  */
 export async function createInferenceBroker(
     signer: JsonRpcSigner | Wallet,
-    contractAddress = ''
+    contractAddress = '',
+    ledger: LedgerBroker
 ): Promise<InferenceBroker> {
-    const broker = new InferenceBroker(signer, contractAddress)
+    const broker = new InferenceBroker(signer, contractAddress, ledger)
     try {
         await broker.initialize()
         return broker

src.ts/inference/broker/request.ts

@@ -66,6 +66,7 @@ export class RequestProcessor extends ZGServingUserBrokerBase {
         svcName: string,
         content: string
     ): Promise<ServingRequestHeaders> {
+
         const headers = await this.getHeader(
             providerAddress,
             svcName,
@@ -74,4 +75,14 @@ export class RequestProcessor extends ZGServingUserBrokerBase {
         )
         return headers
     }
+
+    async calculateInputFee(
+        providerAddress: string,
+        svcName: string,
+        content: string
+    ): Promise<bigint> {
+        const extractor = await this.getExtractor(providerAddress, svcName)
+        const inputFee = await this.calculateInputFees(extractor, content)
+        return inputFee
+    }
 }

src.ts/inference/broker/response.ts

@@ -89,7 +89,7 @@ export class ResponseProcessor extends ZGServingUserBrokerBase {
         try {
             let extractor: Extractor
             extractor = await this.getExtractor(providerAddress, svcName)
-            const outputFee = await this.calculateOutputFees(extractor, content)
+            const outputFee = await this._calculateOutputFees(extractor, content)
 
             await this.settleFee(providerAddress, svcName, outputFee)
 
@@ -139,12 +139,23 @@ export class ResponseProcessor extends ZGServingUserBrokerBase {
         }
     }
 
-    private async calculateOutputFees(
+    private async _calculateOutputFees(
         extractor: Extractor,
         content: string
     ): Promise<bigint> {
         const svc = await extractor.getSvcInfo()
         const outputCount = await extractor.getOutputCount(content)
         return BigInt(outputCount) * svc.outputPrice
     }
+
+    public async calculateOutputFees(
+        providerAddress: string,
+        svcName: string,
+        content: string
+    ): Promise<bigint> {
+        const extractor = await this.getExtractor(providerAddress, svcName)
+        const svc = await extractor.getSvcInfo()
+        const outputCount = await extractor.getOutputCount(content)
+        return BigInt(outputCount) * svc.outputPrice
+    }
 }