From 67d45234f8a2d25d4dfad7e1e2b69626e4a4208b Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 28 Oct 2023 12:22:15 -0700 Subject: [PATCH 1/4] Add palm env fallback to config --- packages/config/src/config.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/config/src/config.ts b/packages/config/src/config.ts index 92331c6b31..2b40eba8c3 100644 --- a/packages/config/src/config.ts +++ b/packages/config/src/config.ts @@ -38,6 +38,9 @@ export const DEFAULT_USER_TOKEN = export const STANDALONE = getVarForEnvironment('STANDALONE') === 'true' || false export const PRODUCTION = getVarForEnvironment('PRODUCTION') === 'true' export const DEFAULT_OPENAI_KEY = getVarForEnvironment('DEFAULT_OPENAI_KEY') +export const DEFAULT_GOOGLEAI_API_KEY = getVarForEnvironment( + 'DEFAULT_GOOGLEAI_API_KEY' +) export const DONT_CRASH_ON_ERROR = getVarForEnvironment('DONT_CRASH_ON_ERROR') === 'true' export const SERVER_PORT = getVarForEnvironment('PORT') || '3030' From a0a42cc40958a51ad4d909ec1d68b3b7aad4d013 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 28 Oct 2023 12:22:53 -0700 Subject: [PATCH 2/4] Improve error outputting from spell runner and manager --- packages/core/shared/src/spellManager/SpellManager.ts | 2 +- packages/core/shared/src/spellManager/SpellRunner.ts | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/packages/core/shared/src/spellManager/SpellManager.ts b/packages/core/shared/src/spellManager/SpellManager.ts index b33b846d34..a70b3de65d 100644 --- a/packages/core/shared/src/spellManager/SpellManager.ts +++ b/packages/core/shared/src/spellManager/SpellManager.ts @@ -90,7 +90,7 @@ export default class SpellManager { this.logger.debug(`Reloading spell ${spellId}`) return this.load(spell) } catch (error) { - this.logger.error(`Error loading spell ${spellId}`, error) + this.logger.error(`Error loading spell ${spellId}: %o`, error) return } } diff --git a/packages/core/shared/src/spellManager/SpellRunner.ts b/packages/core/shared/src/spellManager/SpellRunner.ts index 34f407ba0b..a1595c4ffb 100644 --- a/packages/core/shared/src/spellManager/SpellRunner.ts +++ b/packages/core/shared/src/spellManager/SpellRunner.ts @@ -269,7 +269,7 @@ class SpellRunner { } error(message: string, error: unknown | null = null) { this.busy = false - this.logger.error(message, error) + this.logger.error(message, 'error: %o', error) if (error) throw error throw new Error(message) } @@ -359,6 +359,8 @@ class SpellRunner { this.logger.warn(`No suitable fallback found for ${firstInput}.`) } + this.logger.info('fallback: %o', fallback) + const [node, name] = fallback triggeredNode = node @@ -375,7 +377,7 @@ class SpellRunner { // A run shouldnt take this long. This is a hacl but we are replacing all this soon. setTimeout(() => { this.busy = false - }, 10000) + }, 120000) // this running is where the main "work" happens. // I do wonder whether we could make this even more elegant by having the node @@ -388,7 +390,7 @@ class SpellRunner { this.busy = false return this.outputData } catch (error) { - this.error(`Error loading spell ${this.currentSpell.id}`, error) + this.error(`Error running spell ${this.currentSpell.id} %o`, error) } } } From 687810c64589eaf9b47ceab946e0f63a295f24d6 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 28 Oct 2023 12:23:04 -0700 Subject: [PATCH 3/4] use palm fallback in google ai plugin --- .../server/src/functions/makeChatCompletion.ts | 16 +++++++++++++++- .../server/src/functions/makeTextCompletion.ts | 16 +++++++++++++++- .../server/src/functions/makeTextEmbedding.ts | 4 +++- 3 files changed, 33 insertions(+), 3 deletions(-) diff --git a/packages/plugins/googleai/server/src/functions/makeChatCompletion.ts b/packages/plugins/googleai/server/src/functions/makeChatCompletion.ts index 08dd64e39f..37df308d1f 100644 --- a/packages/plugins/googleai/server/src/functions/makeChatCompletion.ts +++ b/packages/plugins/googleai/server/src/functions/makeChatCompletion.ts @@ -3,6 +3,7 @@ import { CompletionHandlerInputData, saveRequest } from '@magickml/core' import { GOOGLEAI_ENDPOINT } from '../constants' import { trackGoogleAIUsage } from '@magickml/server-core' import { wordCount } from './shared' +import { DEFAULT_GOOGLEAI_API_KEY } from '@magickml/config' type ChatMessage = { author?: string @@ -55,9 +56,22 @@ export async function makeChatCompletion( top_k: parseFloat((node?.data?.top_k as string) ?? '40'), } as any + const apiKey = + (context?.module?.secrets && + context?.module?.secrets['googleai_api_key']) || + DEFAULT_GOOGLEAI_API_KEY || + null + + if (!apiKey) { + return { + success: false, + error: 'GoogleAI API key is required to make a chat completion', + } + } + try { const start = Date.now() - const endpoint = `${GOOGLEAI_ENDPOINT}/${node?.data?.model}:generateMessage?key=${context.module?.secrets?.['googleai_api_key']}` + const endpoint = `${GOOGLEAI_ENDPOINT}/${node?.data?.model}:generateMessage?key=${apiKey}` // Make the API call to GoogleAI const completion = await fetch(endpoint, { method: 'POST', diff --git a/packages/plugins/googleai/server/src/functions/makeTextCompletion.ts b/packages/plugins/googleai/server/src/functions/makeTextCompletion.ts index 394d93fccb..aa758e359f 100644 --- a/packages/plugins/googleai/server/src/functions/makeTextCompletion.ts +++ b/packages/plugins/googleai/server/src/functions/makeTextCompletion.ts @@ -3,6 +3,7 @@ import { CompletionHandlerInputData, saveRequest } from '@magickml/core' import { GOOGLEAI_ENDPOINT } from '../constants' import { trackGoogleAIUsage } from '@magickml/server-core' import { wordCount } from './shared' +import { DEFAULT_GOOGLEAI_API_KEY } from '@magickml/config' /** * Makes an API request to an AI text completion service. @@ -50,8 +51,21 @@ export async function makeTextCompletion( // Make the API request and handle the response. const start = Date.now() + const apiKey = + (context?.module?.secrets && + context?.module?.secrets['googleai_api_key']) || + DEFAULT_GOOGLEAI_API_KEY || + null + + if (!apiKey) { + return { + success: false, + error: 'GoogleAI API key is required to make a text completion', + } + } + try { - const endpoint = `${GOOGLEAI_ENDPOINT}/${node?.data?.model}:generateText?key=${context.module?.secrets?.['googleai_api_key']}` + const endpoint = `${GOOGLEAI_ENDPOINT}/${node?.data?.model}:generateText?key=${apiKey}` // Make the API call to GoogleAI const completion = await fetch(endpoint, { method: 'POST', diff --git a/packages/plugins/googleai/server/src/functions/makeTextEmbedding.ts b/packages/plugins/googleai/server/src/functions/makeTextEmbedding.ts index a5c44f42e8..bd5829bc6a 100644 --- a/packages/plugins/googleai/server/src/functions/makeTextEmbedding.ts +++ b/packages/plugins/googleai/server/src/functions/makeTextEmbedding.ts @@ -7,6 +7,7 @@ import { import { GOOGLEAI_ENDPOINT } from '../constants' import { trackGoogleAIUsage } from '@magickml/server-core' import { wordCount } from './shared' +import { DEFAULT_GOOGLEAI_API_KEY } from '@magickml/config' /** * A function that makes a request to create a text embedding using GoogleAI's @@ -40,6 +41,7 @@ export async function makeTextEmbedding( const apiKey = (context?.module?.secrets && context?.module?.secrets['googleai_api_key']) || + DEFAULT_GOOGLEAI_API_KEY || null if (!apiKey) { @@ -57,7 +59,7 @@ export async function makeTextEmbedding( text: input, } - const endpoint = `${GOOGLEAI_ENDPOINT}/${node?.data?.model}:embedText?key=${context.module?.secrets?.['googleai_api_key']}` + const endpoint = `${GOOGLEAI_ENDPOINT}/${node?.data?.model}:embedText?key=${apiKey}` // Make the API call to GoogleAI const completion = await fetch(endpoint, { method: 'POST', From 4cf8d45cd1c0228c4893532fc84318daf972ee69 Mon Sep 17 00:00:00 2001 From: Michael Date: Sat, 28 Oct 2023 12:39:27 -0700 Subject: [PATCH 4/4] Add default busy timeout for spellrunner env --- packages/config/src/config.ts | 6 ++++++ packages/core/shared/src/spellManager/SpellRunner.ts | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/config/src/config.ts b/packages/config/src/config.ts index 2b40eba8c3..8f78a2ddda 100644 --- a/packages/config/src/config.ts +++ b/packages/config/src/config.ts @@ -141,3 +141,9 @@ export const MANAGER_WARM_UP_MSEC = Number(getVarForEnvironment('MANAGER_WARM_UP_MSEC')) || 5000 export const API_ACCESS_KEY = getVarForEnvironment('API_ACCESS_KEY') || 'apiKey' + +export const SPELLRUNNER_BUSY_TIMEOUT_MSEC = getVarForEnvironment( + 'SPELLRUNNER_BUSY_TIMEOUT_MS' +) + ? Number(getVarForEnvironment('SPELLRUNNER_BUSY_TIMEOUT_MS')) + : 120000 diff --git a/packages/core/shared/src/spellManager/SpellRunner.ts b/packages/core/shared/src/spellManager/SpellRunner.ts index a1595c4ffb..96f23aaba2 100644 --- a/packages/core/shared/src/spellManager/SpellRunner.ts +++ b/packages/core/shared/src/spellManager/SpellRunner.ts @@ -15,6 +15,7 @@ import { extractModuleInputKeys } from './graphHelpers' import SpellManager from './SpellManager' import { getLogger } from '../logger' import { NodeData } from 'rete/types/core/data' +import { SPELLRUNNER_BUSY_TIMEOUT_MSEC } from '@magickml/config' export type RunComponentArgs = { inputs: MagickSpellInput @@ -377,7 +378,7 @@ class SpellRunner { // A run shouldnt take this long. This is a hacl but we are replacing all this soon. setTimeout(() => { this.busy = false - }, 120000) + }, SPELLRUNNER_BUSY_TIMEOUT_MSEC) // this running is where the main "work" happens. // I do wonder whether we could make this even more elegant by having the node