feat: spell scanning

grimoire/framework/src/cli/mod.ts

@@ -1,16 +1,16 @@
-import { knowledgeModule } from '@magickml/knowledge'
-import { nodeModule } from '@magickml/nodes'
-import { portalModule } from '@magickml/portal'
-import { schemasModule } from '@magickml/schemas'
+// import { knowledgeModule } from '@magickml/knowledge'
+// import { nodeModule } from '@magickml/nodes'
+// import { portalModule } from '@magickml/portal'
+// import { schemasModule } from '@magickml/schemas'
 import { spellsModule } from '@magickml/spells'
-import { toolsModule } from '@magickml/tools'
+// import { toolsModule } from '@magickml/tools'
 import type { NitroModule } from 'nitro/types'
 
 export const modules: NitroModule[] = [
-  knowledgeModule,
-  nodeModule,
-  portalModule,
-  schemasModule,
+  // knowledgeModule,
+  // nodeModule,
+  // portalModule,
+  // schemasModule,
   spellsModule,
-  toolsModule,
+  // toolsModule,
 ]

grimoire/modules/nodes/src/runtime/plugins/index.ts

@@ -3,6 +3,7 @@ import { nodeRegistry } from '../utils'
 import { useRuntimeConfig } from 'nitro/runtime'
 import type { NodeFeatures } from '../../features'
 import { getVirtualNodes } from '../exports'
+import { nitro } from 'node:process'
 
 export default defineNovaPlugin<NodeFeatures, any>({
   useRuntimeConfig,

grimoire/modules/spells/package.json

@@ -24,7 +24,8 @@
     "@magickml/behave-graph": "^0.14.15",
     "consola": "^3.2.3",
     "md5": "^2.3.0",
-    "nitro": "npm:nitro-nightly@3x"
+    "nitro": "npm:nitro-nightly@3x",
+    "unimport": "^3.7.2"
   },
   "devDependencies": {
     "@types/md5": "^2.3.5",

grimoire/modules/spells/src/features/index.ts

@@ -1,5 +1,8 @@
 export const spellFeatures = {
-  spells: "spells",
-} as const;
+  // spells: "spells", //
+} as const
 
-export type SpellFeatures = typeof spellFeatures;
+export type SpellFeatures = typeof spellFeatures
+
+export * from './scan-json'
+export * from './rollup-json'

grimoire/modules/spells/src/features/rollup-json.ts

@@ -0,0 +1,55 @@
+import { virtual } from '@gtc-nova/kit/vendor'
+import { fileURLToPath } from 'node:url'
+import type { Nitro } from 'nitro/types'
+import { normalize } from 'path'
+
+export const runtimeDir = fileURLToPath(
+  new URL('dist/runtime/', import.meta.url)
+)
+
+interface SpellDefinition {
+  path: string
+  data: any
+  name: string
+}
+
+/**
+ * Creates a Rollup plugin for JSON data.
+ * @param moduleName The name of the module.
+ * @param spells An array of spell definitions.
+ * @returns A function that creates the Rollup plugin.
+ */
+export function createRollupJsonPlugin(
+  moduleName: string,
+  spells: SpellDefinition[]
+) {
+  return function (nitro: Nitro) {
+    const generateSpellsCode = (): string => {
+      const imports = spells.map(
+        (spell, index) => `import spell${index} from '${spell.path}';`
+      )
+      const spellsArray = spells.map(
+        (spell, index) => `
+        {
+          path: ${JSON.stringify(normalize(spell.path))},
+          data: spell${index},
+          name: ${JSON.stringify(spell.name)}
+        }`
+      )
+
+      return `
+${imports.join('\n')}
+
+export const spells = [
+${spellsArray.join(',\n')}
+];
+      `.trim()
+    }
+
+    const virtualFiles: Record<string, () => string> = {
+      [`#${moduleName}-virtual/spells`]: generateSpellsCode,
+    }
+
+    return virtual(virtualFiles, nitro.vfs)
+  }
+}

grimoire/modules/spells/src/features/scan-json.ts

@@ -0,0 +1,42 @@
+import { normalize, join } from 'pathe'
+import fg from 'fast-glob'
+import { readFile } from 'fs/promises'
+
+type ScanJsonFilesOptions = {
+  fileFilter?: (file: string) => boolean
+  filePatterns?: string[]
+  cwd?: string
+}
+
+export async function scanJsonFilesFromDir(
+  dir: string | string[],
+  options?: ScanJsonFilesOptions
+): Promise<string[]> {
+  const dirs = (Array.isArray(dir) ? dir : [dir]).map(d => normalize(d))
+  const fileFilter = options?.fileFilter || (() => true)
+  const filePatterns = options?.filePatterns || ['*.spell.json']
+
+  const result = await Promise.all(
+    dirs.map(async i => {
+      const patterns = [i, ...filePatterns.map(p => join(i, p))]
+      return fg(patterns, {
+        absolute: true,
+        cwd: options?.cwd || process.cwd(),
+        onlyFiles: true,
+        followSymbolicLinks: true,
+      })
+    })
+  )
+
+  const allFiles = result
+    .flat()
+    .map(f => normalize(f))
+    .sort()
+
+  return Array.from(new Set(allFiles)).filter(fileFilter)
+}
+
+export async function readJsonFile(filePath: string): Promise<any> {
+  const content = await readFile(filePath, 'utf-8')
+  return JSON.parse(content)
+}

grimoire/modules/spells/src/index.ts

@@ -1,17 +1,60 @@
 import { defineNovaModule } from '@gtc-nova/kit'
-import { spellFeatures, type SpellFeatures } from './features'
+import { normalize } from 'pathe'
+import {
+  scanJsonFilesFromDir,
+  readJsonFile,
+  createRollupJsonPlugin,
+} from './features'
+import { createRollupPlugin } from '@gtc-nova/kit/rollup'
+import type { Import } from 'unimport'
 
-export const spellsModule = defineNovaModule<SpellFeatures>({
+const f = {
+  spells: 'spells',
+}
+
+export const spellsModule = defineNovaModule<{}>({
   name: 'spells',
-  features: spellFeatures,
-  featureTypeFunctions: {
-    spells: () => {
-      console.log('spells')
-    },
-  },
-  pluginsDir: './../src/runtime/plugins',
+  features: {},
+  featureTypeFunctions: {},
   metaUrl: import.meta.url,
+  pluginsDir: './../src/runtime/plugins',
+
   hooks: [],
+  async setup(nitro) {
+    const spellFiles = await scanJsonFilesFromDir('spells')
+    const spellJSON = await Promise.all(
+      spellFiles.map(async spellFile => {
+        const spellData = await readJsonFile(spellFile)
+        return {
+          path: normalize(spellFile),
+          data: spellData,
+          name: normalize(spellFile).split('/').pop() as string,
+        }
+      })
+    )
+
+    const scannedSpells: Import[] = spellJSON.map(spell => ({
+      name: spell.name as string,
+      from: spell.path,
+      as: 'default',
+    }))
+
+    // @ts-ignore
+    nitro['scannedSpells'] = scannedSpells
+
+    nitro.hooks.hook('rollup:before', async (nit, config) => {
+      // @ts-ignore
+      config.plugins.push(createRollupJsonPlugin(this.name, spellJSON)(nitro))
+    })
+    // nitro.hooks.hook('rollup:before', async (nitro, rollupConfig) => {
+    //   const rollupPlugin = createRollupPlugin('spells', ['spells'])
+
+    //   // @ts-ignore
+    //   rollupConfig.plugins?.push(rollupPlugin)
+    // })
+
+    console.log('Scanned spells:', scannedSpells)
+  },
 })
 
 export type { Spell, SerializedSpell, SpellRegistry } from './types'

grimoire/modules/spells/src/runtime/exports.ts

@@ -1,7 +1,12 @@
 // @ts-ignore
-import { handlers as spells } from "#spells-virtual/spells";
+import { spells } from '#spells-virtual/spells'
 
-import type { BaseVirtualHandler } from "@gtc-nova/kit/runtime";
-import type { Spell } from "../types";
+import type { Spell } from '../types'
 
-export const getVirtualSpells = (): BaseVirtualHandler<Spell>[] => spells;
+interface ScannedSpell {
+  path: string
+  data: Spell
+  name: string
+}
+
+export const getVirtualSpells = (): ScannedSpell[] => spells

grimoire/modules/spells/src/runtime/plugins/spells-plugin.ts

@@ -1,65 +1,29 @@
 import { defineNovaPlugin } from '@gtc-nova/kit/runtime'
-import { spellRegistry } from '../utils/registry'
+import { SpellRegistryManager } from '../utils/registry'
 import { useRuntimeConfig } from 'nitro/runtime'
 import type { SpellFeatures } from '../../features'
 import { getVirtualSpells } from '../exports'
-import { readGraphFromJSON, writeGraphToJSON } from '@magickml/behave-graph'
-import type { Spell, SerializedSpell } from '../../types'
-import fs from 'fs/promises'
-import path from 'path'
 
-export default defineNovaPlugin<SpellFeatures, any, any, any, any>({
+export default defineNovaPlugin<SpellFeatures, any>({
   useRuntimeConfig,
   initialize: (nitro, config) => {
     const spellOptions = config.spells || {}
     return { spellOptions }
   },
-  before: async (nitro, br) => {},
-  runtimeSetup: {
-    spells: {
-      getVirtualHandlers: getVirtualSpells,
-      initFeatureHandlers: async (nitro, handlers) => {
-        for (const handler of handlers) {
-          const spellDefinition = (await handler.handler()).default
-          console.log('feature spell scanned:', spellDefinition)
-        }
-        // const spellsDir = path.join(process.cwd(), 'server', 'spells');
-        // await loadSpellsFromDirectory(spellsDir);
-        // for (const handler of handlers) {
-        //   const spellDefinition = (await handler.handler()).default;
-        //   spellRegistry.add(spellDefinition);
-        // }
-      },
-    },
+  before: async (nitro, br) => {
+    nitro.spellRegistry = new SpellRegistryManager()
+    const virtualSpells = getVirtualSpells()
+
+    console.log('Adding virtual spells to registry')
+
+    for (const spell of virtualSpells) {
+      nitro.spellRegistry.add(spell.data)
+    }
   },
+  runtimeSetup: {}, // Since spells are serialized, we can skip this step
   after: (nitro, br) => {
-    // @ts-ignore todo: declare
-    nitro.spellRegistry = spellRegistry
-    console.info(
-      'Spell module initialized. Use nitro.spellRegistry to access spells.'
-    )
+    nitro.spellRegistry.getAll().forEach(spell => {
+      console.log(spell)
+    })
   },
 })
-
-async function loadSpellsFromDirectory(directory: string) {
-  try {
-    const files = await fs.readdir(directory)
-    for (const file of files) {
-      if (path.extname(file) === '.json') {
-        const filePath = path.join(directory, file)
-        const content = await fs.readFile(filePath, 'utf-8')
-        const serializedSpell: SerializedSpell = JSON.parse(content)
-        const spell: Spell = {
-          ...serializedSpell,
-          graph: readGraphFromJSON({
-            graphJson: serializedSpell.graph,
-            registry: useRuntimeConfig().registry,
-          }),
-        }
-        spellRegistry.add(spell)
-      }
-    }
-  } catch (error) {
-    console.error('Error loading spells from directory:', error)
-  }
-}

grimoire/modules/spells/src/runtime/utils/registry.ts

@@ -1,7 +1,7 @@
 import type { Spell, SpellRegistry } from '../../types'
 
 // This is a mock of our existing system
-class SpellRegistryManager implements SpellRegistry {
+export class SpellRegistryManager implements SpellRegistry {
   private spells: Map<string, Spell> = new Map()
 
   get(id: string): Spell | undefined {