From ee3ca582a5447b1be978ff78f2eacc8984a023f4 Mon Sep 17 00:00:00 2001 From: Simeon Widdis Date: Thu, 1 Feb 2024 13:29:50 -0800 Subject: [PATCH] Add Index-based adaptor for integrations (#1381) * Update comment for json adaptor construction Signed-off-by: Simeon Widdis * Stub index data adaptor class Signed-off-by: Simeon Widdis * Add initial impl for findIntegrationVersions Signed-off-by: Simeon Widdis * Fill in simple getDirectoryType implementation Signed-off-by: Simeon Widdis * Implement index adaptor as wrapper for json adaptor Signed-off-by: Simeon Widdis * Add integration template type for index Signed-off-by: Simeon Widdis * Fix lints for server/routes Signed-off-by: Simeon Widdis * Fix integrations_manager lints Signed-off-by: Simeon Widdis * Refactor template manager to support multiple readers at once Signed-off-by: Simeon Widdis * Rename FileSystemCatalogDataAdaptor -> FileSystemDataAdaptor Signed-off-by: Simeon Widdis * Add IndexReader to existing Manager logic Signed-off-by: Simeon Widdis * Fix plugin label type Signed-off-by: Simeon Widdis * Add tests for index adaptor Signed-off-by: Simeon Widdis * Add object management to integration objects Signed-off-by: Simeon Widdis * Fix bug with version parsing for numeric integration names Signed-off-by: Simeon Widdis * Prioritize dynamic integrations over defaults Signed-off-by: Simeon Widdis --------- Signed-off-by: Simeon Widdis --- .../__test__/json_repository.test.ts | 21 ++-- .../__test__/local_fs_repository.test.ts | 14 +-- .../integrations/integrations_manager.ts | 34 +++--- .../__test__/index_data_adaptor.test.ts | 105 ++++++++++++++++++ .../__test__/json_data_adaptor.test.ts | 16 ++- .../repository/__test__/repository.test.ts | 3 +- .../repository/fs_data_adaptor.ts | 6 +- .../repository/index_data_adaptor.ts | 56 ++++++++++ .../repository/integration_reader.ts | 16 +-- .../repository/json_data_adaptor.ts | 7 +- .../integrations/repository/repository.ts | 58 +++++++--- server/adaptors/integrations/types.ts | 8 ++ server/plugin.ts | 81 +++++++++++++- .../__tests__/integrations_router.test.ts | 6 +- .../integrations/integrations_router.ts | 33 +++--- 15 files changed, 368 insertions(+), 96 deletions(-) create mode 100644 server/adaptors/integrations/repository/__test__/index_data_adaptor.test.ts create mode 100644 server/adaptors/integrations/repository/index_data_adaptor.ts diff --git a/server/adaptors/integrations/__test__/json_repository.test.ts b/server/adaptors/integrations/__test__/json_repository.test.ts index 0e777b3eca..0b979e31ff 100644 --- a/server/adaptors/integrations/__test__/json_repository.test.ts +++ b/server/adaptors/integrations/__test__/json_repository.test.ts @@ -42,10 +42,9 @@ describe('The Local Serialized Catalog', () => { it('Should pass deep validation for all serialized integrations', async () => { const serialized = await fetchSerializedIntegrations(); - const repository = new TemplateManager( - '.', - new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]) - ); + const repository = new TemplateManager([ + new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]), + ]); for (const integ of await repository.getIntegrationList()) { const validationResult = await deepCheck(integ); @@ -55,10 +54,9 @@ describe('The Local Serialized Catalog', () => { it('Should correctly retrieve a logo', async () => { const serialized = await fetchSerializedIntegrations(); - const repository = new TemplateManager( - '.', - new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]) - ); + const repository = new TemplateManager([ + new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]), + ]); const integration = (await repository.getIntegration('nginx')) as IntegrationReader; const logoStatic = await integration.getStatic('logo.svg'); @@ -68,10 +66,9 @@ describe('The Local Serialized Catalog', () => { it('Should correctly retrieve a gallery image', async () => { const serialized = await fetchSerializedIntegrations(); - const repository = new TemplateManager( - '.', - new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]) - ); + const repository = new TemplateManager([ + new JsonCatalogDataAdaptor(serialized.value as SerializedIntegration[]), + ]); const integration = (await repository.getIntegration('nginx')) as IntegrationReader; const logoStatic = await integration.getStatic('dashboard1.png'); diff --git a/server/adaptors/integrations/__test__/local_fs_repository.test.ts b/server/adaptors/integrations/__test__/local_fs_repository.test.ts index dcacf02bbb..3a332ec54e 100644 --- a/server/adaptors/integrations/__test__/local_fs_repository.test.ts +++ b/server/adaptors/integrations/__test__/local_fs_repository.test.ts @@ -12,6 +12,11 @@ import { IntegrationReader } from '../repository/integration_reader'; import path from 'path'; import * as fs from 'fs/promises'; import { deepCheck } from '../repository/utils'; +import { FileSystemDataAdaptor } from '../repository/fs_data_adaptor'; + +const repository: TemplateManager = new TemplateManager([ + new FileSystemDataAdaptor(path.join(__dirname, '../__data__/repository')), +]); describe('The local repository', () => { it('Should only contain valid integration directories or files.', async () => { @@ -32,9 +37,6 @@ describe('The local repository', () => { }); it('Should pass deep validation for all local integrations.', async () => { - const repository: TemplateManager = new TemplateManager( - path.join(__dirname, '../__data__/repository') - ); const integrations: IntegrationReader[] = await repository.getIntegrationList(); await Promise.all( integrations.map(async (i) => { @@ -50,18 +52,12 @@ describe('The local repository', () => { describe('Local Nginx Integration', () => { it('Should serialize without errors', async () => { - const repository: TemplateManager = new TemplateManager( - path.join(__dirname, '../__data__/repository') - ); const integration = await repository.getIntegration('nginx'); await expect(integration?.serialize()).resolves.toHaveProperty('ok', true); }); it('Should serialize to include the config', async () => { - const repository: TemplateManager = new TemplateManager( - path.join(__dirname, '../__data__/repository') - ); const integration = await repository.getIntegration('nginx'); const config = await integration!.getConfig(); const serialized = await integration!.serialize(); diff --git a/server/adaptors/integrations/integrations_manager.ts b/server/adaptors/integrations/integrations_manager.ts index c516d3fc78..3bbda134d8 100644 --- a/server/adaptors/integrations/integrations_manager.ts +++ b/server/adaptors/integrations/integrations_manager.ts @@ -9,6 +9,8 @@ import { IntegrationsAdaptor } from './integrations_adaptor'; import { SavedObject, SavedObjectsClientContract } from '../../../../../src/core/server/types'; import { IntegrationInstanceBuilder } from './integrations_builder'; import { TemplateManager } from './repository/repository'; +import { FileSystemDataAdaptor } from './repository/fs_data_adaptor'; +import { IndexDataAdaptor } from './repository/index_data_adaptor'; export class IntegrationsManager implements IntegrationsAdaptor { client: SavedObjectsClientContract; @@ -18,21 +20,25 @@ export class IntegrationsManager implements IntegrationsAdaptor { constructor(client: SavedObjectsClientContract, repository?: TemplateManager) { this.client = client; this.repository = - repository ?? new TemplateManager(path.join(__dirname, '__data__/repository')); + repository ?? + new TemplateManager([ + new IndexDataAdaptor(this.client), + new FileSystemDataAdaptor(path.join(__dirname, '__data__/repository')), + ]); this.instanceBuilder = new IntegrationInstanceBuilder(this.client); } deleteIntegrationInstance = async (id: string): Promise => { - let children: any; + let children: SavedObject; try { children = await this.client.get('integration-instance', id); - } catch (err: any) { + } catch (err) { return err.output?.statusCode === 404 ? Promise.resolve([id]) : Promise.reject(err); } const toDelete = children.attributes.assets - .filter((i: any) => i.assetId) - .map((i: any) => { + .filter((i: AssetReference) => i.assetId) + .map((i: AssetReference) => { return { id: i.assetId, type: i.assetType }; }); toDelete.push({ id, type: 'integration-instance' }); @@ -43,7 +49,7 @@ export class IntegrationsManager implements IntegrationsAdaptor { try { await this.client.delete(asset.type, asset.id); return Promise.resolve(asset.id); - } catch (err: any) { + } catch (err) { addRequestToMetric('integrations', 'delete', err); return err.output?.statusCode === 404 ? Promise.resolve(asset.id) : Promise.reject(err); } @@ -101,20 +107,22 @@ export class IntegrationsManager implements IntegrationsAdaptor { query?: IntegrationInstanceQuery ): Promise => { addRequestToMetric('integrations', 'get', 'count'); - const result = await this.client.get('integration-instance', `${query!.id}`); + const result = (await this.client.get('integration-instance', `${query!.id}`)) as SavedObject< + IntegrationInstance + >; return Promise.resolve(this.buildInstanceResponse(result)); }; buildInstanceResponse = async ( - savedObj: SavedObject + savedObj: SavedObject ): Promise => { - const assets: AssetReference[] | undefined = (savedObj.attributes as any)?.assets; + const assets: AssetReference[] | undefined = savedObj.attributes.assets; const status: string = assets ? await this.getAssetStatus(assets) : 'available'; return { id: savedObj.id, status, - ...(savedObj.attributes as any), + ...savedObj.attributes, }; }; @@ -124,7 +132,7 @@ export class IntegrationsManager implements IntegrationsAdaptor { try { await this.client.get(asset.assetType, asset.assetId); return { id: asset.assetId, status: 'available' }; - } catch (err: any) { + } catch (err) { const statusCode = err.output?.statusCode; if (statusCode && 400 <= statusCode && statusCode < 500) { return { id: asset.assetId, status: 'unavailable' }; @@ -166,7 +174,7 @@ export class IntegrationsManager implements IntegrationsAdaptor { }); const test = await this.client.create('integration-instance', result); return Promise.resolve({ ...result, id: test.id }); - } catch (err: any) { + } catch (err) { addRequestToMetric('integrations', 'create', err); return Promise.reject({ message: err.message, @@ -213,7 +221,7 @@ export class IntegrationsManager implements IntegrationsAdaptor { }); }; - getAssets = async (templateName: string): Promise<{ savedObjects?: any }> => { + getAssets = async (templateName: string): Promise => { const integration = await this.repository.getIntegration(templateName); if (integration === null) { return Promise.reject({ diff --git a/server/adaptors/integrations/repository/__test__/index_data_adaptor.test.ts b/server/adaptors/integrations/repository/__test__/index_data_adaptor.test.ts new file mode 100644 index 0000000000..6ab7f77b73 --- /dev/null +++ b/server/adaptors/integrations/repository/__test__/index_data_adaptor.test.ts @@ -0,0 +1,105 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { IntegrationReader } from '../integration_reader'; +import { JsonCatalogDataAdaptor } from '../json_data_adaptor'; +import { TEST_INTEGRATION_CONFIG } from '../../../../../test/constants'; +import { savedObjectsClientMock } from '../../../../../../../src/core/server/mocks'; +import { IndexDataAdaptor } from '../index_data_adaptor'; +import { SavedObjectsClientContract } from '../../../../../../../src/core/server'; + +// Simplified catalog for integration searching -- Do not use for full deserialization tests. +const TEST_CATALOG_NO_SERIALIZATION: SerializedIntegration[] = [ + { + ...(TEST_INTEGRATION_CONFIG as SerializedIntegration), + name: 'sample1', + }, + { + ...(TEST_INTEGRATION_CONFIG as SerializedIntegration), + name: 'sample2', + }, + { + ...(TEST_INTEGRATION_CONFIG as SerializedIntegration), + name: 'sample2', + version: '2.1.0', + }, +]; + +// Copy of json_data_adaptor.test.ts with new reader type +// Since implementation at time of writing is to defer to json adaptor +describe('Index Data Adaptor', () => { + let mockClient: SavedObjectsClientContract; + + beforeEach(() => { + mockClient = savedObjectsClientMock.create(); + mockClient.find = jest.fn().mockResolvedValue({ + saved_objects: TEST_CATALOG_NO_SERIALIZATION.map((item) => ({ + attributes: item, + })), + }); + }); + + it('Should correctly identify repository type', async () => { + const adaptor = new IndexDataAdaptor(mockClient); + await expect(adaptor.getDirectoryType()).resolves.toBe('repository'); + }); + + it('Should correctly identify integration type after filtering', async () => { + const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION); + const joined = await adaptor.join('sample1'); + await expect(joined.getDirectoryType()).resolves.toBe('integration'); + }); + + it('Should correctly retrieve integration versions', async () => { + const adaptor = new IndexDataAdaptor(mockClient); + const versions = await adaptor.findIntegrationVersions('sample2'); + expect((versions as { value: string[] }).value).toHaveLength(2); + }); + + it('Should correctly supply latest integration version for IntegrationReader', async () => { + const adaptor = new IndexDataAdaptor(mockClient); + const reader = new IntegrationReader('sample2', adaptor.join('sample2')); + const version = await reader.getLatestVersion(); + expect(version).toBe('2.1.0'); + }); + + it('Should find integration names', async () => { + const adaptor = new IndexDataAdaptor(mockClient); + const integResult = await adaptor.findIntegrations(); + const integs = (integResult as { value: string[] }).value; + integs.sort(); + + expect(integs).toEqual(['sample1', 'sample2']); + }); + + it('Should reject any attempts to read a file with a type', async () => { + const adaptor = new IndexDataAdaptor(mockClient); + const result = await adaptor.readFile('logs-1.0.0.json', 'schemas'); + await expect(result.error?.message).toBe( + 'JSON adaptor does not support subtypes (isConfigLocalized: true)' + ); + }); + + it('Should reject any attempts to read a raw file', async () => { + const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION); + const result = await adaptor.readFileRaw('logo.svg', 'static'); + await expect(result.error?.message).toBe( + 'JSON adaptor does not support raw files (isConfigLocalized: true)' + ); + }); + + it('Should reject nested directory searching', async () => { + const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION); + const result = await adaptor.findIntegrations('sample1'); + await expect(result.error?.message).toBe( + 'Finding integrations for custom dirs not supported for JSONreader' + ); + }); + + it('Should report unknown directory type if integration list is empty', async () => { + const adaptor = new JsonCatalogDataAdaptor([]); + await expect(adaptor.getDirectoryType()).resolves.toBe('unknown'); + }); +}); diff --git a/server/adaptors/integrations/repository/__test__/json_data_adaptor.test.ts b/server/adaptors/integrations/repository/__test__/json_data_adaptor.test.ts index 8c703b7516..434af5074b 100644 --- a/server/adaptors/integrations/repository/__test__/json_data_adaptor.test.ts +++ b/server/adaptors/integrations/repository/__test__/json_data_adaptor.test.ts @@ -8,6 +8,7 @@ import { IntegrationReader } from '../integration_reader'; import path from 'path'; import { JsonCatalogDataAdaptor } from '../json_data_adaptor'; import { TEST_INTEGRATION_CONFIG } from '../../../../../test/constants'; +import { FileSystemDataAdaptor } from '../fs_data_adaptor'; // Simplified catalog for integration searching -- Do not use for full deserialization tests. const TEST_CATALOG_NO_SERIALIZATION: SerializedIntegration[] = [ @@ -28,9 +29,9 @@ const TEST_CATALOG_NO_SERIALIZATION: SerializedIntegration[] = [ describe('JSON Data Adaptor', () => { it('Should be able to deserialize a serialized integration', async () => { - const repository: TemplateManager = new TemplateManager( - path.join(__dirname, '../../__data__/repository') - ); + const repository: TemplateManager = new TemplateManager([ + new FileSystemDataAdaptor(path.join(__dirname, '../../__data__/repository')), + ]); const fsIntegration: IntegrationReader = (await repository.getIntegration('nginx'))!; const fsConfig = await fsIntegration.getConfig(); const serialized = await fsIntegration.serialize(); @@ -112,4 +113,13 @@ describe('JSON Data Adaptor', () => { const adaptor = new JsonCatalogDataAdaptor([]); await expect(adaptor.getDirectoryType()).resolves.toBe('unknown'); }); + + // Bug: a previous regex for version finding counted the `8` in `k8s-1.0.0.json` as the version + it('Should correctly read a config with a number in the name', async () => { + const adaptor = new JsonCatalogDataAdaptor(TEST_CATALOG_NO_SERIALIZATION); + await expect(adaptor.readFile('sample2-2.1.0.json')).resolves.toMatchObject({ + ok: true, + value: TEST_CATALOG_NO_SERIALIZATION[2], + }); + }); }); diff --git a/server/adaptors/integrations/repository/__test__/repository.test.ts b/server/adaptors/integrations/repository/__test__/repository.test.ts index 816b44eaa0..ae0698ffad 100644 --- a/server/adaptors/integrations/repository/__test__/repository.test.ts +++ b/server/adaptors/integrations/repository/__test__/repository.test.ts @@ -8,6 +8,7 @@ import { TemplateManager } from '../repository'; import { IntegrationReader } from '../integration_reader'; import { Dirent, Stats } from 'fs'; import path from 'path'; +import { FileSystemDataAdaptor } from '../fs_data_adaptor'; jest.mock('fs/promises'); @@ -15,7 +16,7 @@ describe('Repository', () => { let repository: TemplateManager; beforeEach(() => { - repository = new TemplateManager('path/to/directory'); + repository = new TemplateManager([new FileSystemDataAdaptor('path/to/directory')]); }); afterEach(() => { diff --git a/server/adaptors/integrations/repository/fs_data_adaptor.ts b/server/adaptors/integrations/repository/fs_data_adaptor.ts index 52c5dff6d5..5687749ca2 100644 --- a/server/adaptors/integrations/repository/fs_data_adaptor.ts +++ b/server/adaptors/integrations/repository/fs_data_adaptor.ts @@ -21,7 +21,7 @@ const safeIsDirectory = async (maybeDirectory: string): Promise => { * A CatalogDataAdaptor that reads from the local filesystem. * Used to read default Integrations shipped in the in-product catalog at `__data__`. */ -export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor { +export class FileSystemDataAdaptor implements CatalogDataAdaptor { isConfigLocalized = false; directory: string; @@ -131,7 +131,7 @@ export class FileSystemCatalogDataAdaptor implements CatalogDataAdaptor { return hasSchemas ? 'integration' : 'repository'; } - join(filename: string): FileSystemCatalogDataAdaptor { - return new FileSystemCatalogDataAdaptor(path.join(this.directory, filename)); + join(filename: string): FileSystemDataAdaptor { + return new FileSystemDataAdaptor(path.join(this.directory, filename)); } } diff --git a/server/adaptors/integrations/repository/index_data_adaptor.ts b/server/adaptors/integrations/repository/index_data_adaptor.ts new file mode 100644 index 0000000000..3344dd0720 --- /dev/null +++ b/server/adaptors/integrations/repository/index_data_adaptor.ts @@ -0,0 +1,56 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +import { CatalogDataAdaptor, IntegrationPart } from './catalog_data_adaptor'; +import { SavedObjectsClientContract } from '../../../../../../src/core/server/types'; +import { JsonCatalogDataAdaptor } from './json_data_adaptor'; + +export class IndexDataAdaptor implements CatalogDataAdaptor { + isConfigLocalized = true; + directory?: string; + client: SavedObjectsClientContract; + + constructor(client: SavedObjectsClientContract, directory?: string) { + this.directory = directory; + this.client = client; + } + + private async asJsonAdaptor(): Promise { + const results = await this.client.find({ type: 'integration-template' }); + const filteredIntegrations: SerializedIntegration[] = results.saved_objects + .map((obj) => obj.attributes as SerializedIntegration) + .filter((obj) => this.directory === undefined || this.directory === obj.name); + return new JsonCatalogDataAdaptor(filteredIntegrations); + } + + async findIntegrationVersions(dirname?: string | undefined): Promise> { + const adaptor = await this.asJsonAdaptor(); + return await adaptor.findIntegrationVersions(dirname); + } + + async readFile(filename: string, type?: IntegrationPart): Promise> { + const adaptor = await this.asJsonAdaptor(); + return await adaptor.readFile(filename, type); + } + + async readFileRaw(filename: string, type?: IntegrationPart): Promise> { + const adaptor = await this.asJsonAdaptor(); + return await adaptor.readFileRaw(filename, type); + } + + async findIntegrations(dirname: string = '.'): Promise> { + const adaptor = await this.asJsonAdaptor(); + return await adaptor.findIntegrations(dirname); + } + + async getDirectoryType(dirname?: string): Promise<'integration' | 'repository' | 'unknown'> { + const adaptor = await this.asJsonAdaptor(); + return await adaptor.getDirectoryType(dirname); + } + + join(filename: string): IndexDataAdaptor { + return new IndexDataAdaptor(this.client, filename); + } +} diff --git a/server/adaptors/integrations/repository/integration_reader.ts b/server/adaptors/integrations/repository/integration_reader.ts index 0f28c5d420..ea3acbe77b 100644 --- a/server/adaptors/integrations/repository/integration_reader.ts +++ b/server/adaptors/integrations/repository/integration_reader.ts @@ -6,7 +6,7 @@ import path from 'path'; import semver from 'semver'; import { validateTemplate } from '../validators'; -import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; +import { FileSystemDataAdaptor } from './fs_data_adaptor'; import { CatalogDataAdaptor, IntegrationPart } from './catalog_data_adaptor'; import { foldResults, pruneConfig } from './utils'; @@ -23,7 +23,7 @@ export class IntegrationReader { constructor(directory: string, reader?: CatalogDataAdaptor) { this.directory = directory; this.name = path.basename(directory); - this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); + this.reader = reader ?? new FileSystemDataAdaptor(directory); } /** @@ -178,17 +178,7 @@ export class IntegrationReader { * @param version The version of the integration to retrieve assets for. * @returns An object containing the different types of assets. */ - async getAssets( - version?: string - ): Promise< - Result<{ - savedObjects?: object[]; - queries?: Array<{ - query: string; - language: string; - }>; - }> - > { + async getAssets(version?: string): Promise> { const configResult = await this.getRawConfig(version); if (!configResult.ok) { return configResult; diff --git a/server/adaptors/integrations/repository/json_data_adaptor.ts b/server/adaptors/integrations/repository/json_data_adaptor.ts index 05c0b11104..2ab1547e16 100644 --- a/server/adaptors/integrations/repository/json_data_adaptor.ts +++ b/server/adaptors/integrations/repository/json_data_adaptor.ts @@ -16,7 +16,7 @@ export class JsonCatalogDataAdaptor implements CatalogDataAdaptor { /** * Creates a new FileSystemCatalogDataAdaptor instance. * - * @param directory The base directory from which to read files. This is not sanitized. + * @param integrationsList The list of JSON-serialized integrations to use as a pseudo-directory. */ constructor(integrationsList: SerializedIntegration[]) { this.integrationsList = integrationsList; @@ -41,10 +41,9 @@ export class JsonCatalogDataAdaptor implements CatalogDataAdaptor { }; } - const name = filename.split('-')[0]; - const version = filename.match(/\d+(\.\d+)*/); + const filenameParts = filename.match(/([\w]+)-(\d+(\.\d+)*)\.json/); for (const integ of this.integrationsList) { - if (integ.name === name && integ.version === version?.[0]) { + if (integ.name === filenameParts?.[1] && integ.version === filenameParts?.[2]) { return { ok: true, value: integ }; } } diff --git a/server/adaptors/integrations/repository/repository.ts b/server/adaptors/integrations/repository/repository.ts index 0337372049..1c2359cc05 100644 --- a/server/adaptors/integrations/repository/repository.ts +++ b/server/adaptors/integrations/repository/repository.ts @@ -3,44 +3,66 @@ * SPDX-License-Identifier: Apache-2.0 */ -import * as path from 'path'; import { IntegrationReader } from './integration_reader'; -import { FileSystemCatalogDataAdaptor } from './fs_data_adaptor'; import { CatalogDataAdaptor } from './catalog_data_adaptor'; export class TemplateManager { - reader: CatalogDataAdaptor; - directory: string; + readers: CatalogDataAdaptor[]; - constructor(directory: string, reader?: CatalogDataAdaptor) { - this.directory = directory; - this.reader = reader ?? new FileSystemCatalogDataAdaptor(directory); + constructor(readers: CatalogDataAdaptor[]) { + this.readers = readers; } async getIntegrationList(): Promise { - // TODO in the future, we want to support traversing nested directory structures. - const folders = await this.reader.findIntegrations(); + const lists = await Promise.all( + this.readers.map((reader) => this.getReaderIntegrationList(reader)) + ); + const flattened = lists.flat(); + + // If there are collisions by name, prioritize earlier readers over later ones. + const seen = new Set(); + return flattened.filter((item) => { + if (seen.has(item.name)) { + return false; + } + seen.add(item.name); + return true; + }); + } + + private async getReaderIntegrationList(reader: CatalogDataAdaptor): Promise { + const folders = await reader.findIntegrations(); if (!folders.ok) { - console.error(`Error reading integration directories in: ${this.directory}`, folders.error); return []; } const integrations = await Promise.all( - folders.value.map((i) => - this.getIntegration(path.relative(this.directory, path.join(this.directory, i))) - ) + folders.value.map((integrationName) => this.getReaderIntegration(reader, integrationName)) ); return integrations.filter((x) => x !== null) as IntegrationReader[]; } - async getIntegration(integPath: string): Promise { - if ((await this.reader.getDirectoryType(integPath)) !== 'integration') { - console.error(`Requested integration '${integPath}' does not exist`); + async getIntegration(integrationName: string): Promise { + const maybeIntegrations = await Promise.all( + this.readers.map((reader) => this.getReaderIntegration(reader, integrationName)) + ); + for (const maybeIntegration of maybeIntegrations) { + if (maybeIntegration !== null) { + return maybeIntegration; + } + } + return null; + } + + private async getReaderIntegration( + reader: CatalogDataAdaptor, + integrationName: string + ): Promise { + if ((await reader.getDirectoryType(integrationName)) !== 'integration') { return null; } - const integ = new IntegrationReader(integPath, this.reader.join(integPath)); + const integ = new IntegrationReader(integrationName, reader.join(integrationName)); const checkResult = await integ.getConfig(); if (!checkResult.ok) { - console.error(`Integration '${integPath}' is invalid:`, checkResult.error); return null; } return integ; diff --git a/server/adaptors/integrations/types.ts b/server/adaptors/integrations/types.ts index 5e7565a133..7868faee83 100644 --- a/server/adaptors/integrations/types.ts +++ b/server/adaptors/integrations/types.ts @@ -62,6 +62,14 @@ interface IntegrationAssets { }>; } +interface ParsedIntegrationAssets { + savedObjects?: object[]; + queries?: Array<{ + query: string; + language: string; + }>; +} + interface SerializedIntegrationAssets extends IntegrationAssets { savedObjects?: { name: string; diff --git a/server/plugin.ts b/server/plugin.ts index 8efee24159..303a99e52e 100644 --- a/server/plugin.ts +++ b/server/plugin.ts @@ -12,6 +12,7 @@ import { Logger, Plugin, PluginInitializerContext, + SavedObject, SavedObjectsType, } from '../../../src/core/server'; import { OpenSearchObservabilityPlugin } from './adaptors/opensearch_observability_plugin'; @@ -90,7 +91,10 @@ export class ObservabilityPlugin migrations: { '3.0.0': (doc) => ({ ...doc, description: '' }), '3.0.1': (doc) => ({ ...doc, description: 'Some Description Text' }), - '3.0.2': (doc) => ({ ...doc, dateCreated: parseInt(doc.dateCreated || '0', 10) }), + '3.0.2': (doc) => ({ + ...doc, + dateCreated: parseInt((doc as { dateCreated?: string }).dateCreated || '0', 10), + }), }, }; @@ -98,6 +102,18 @@ export class ObservabilityPlugin name: 'integration-instance', hidden: false, namespaceType: 'single', + management: { + importableAndExportable: true, + getInAppUrl(obj: SavedObject) { + return { + path: `/app/integrations#/installed/${obj.id}`, + uiCapabilitiesPath: 'advancedSettings.show', + }; + }, + getTitle(obj: SavedObject) { + return obj.attributes.name; + }, + }, mappings: { dynamic: false, properties: { @@ -120,8 +136,71 @@ export class ObservabilityPlugin }, }; + const integrationTemplateType: SavedObjectsType = { + name: 'integration-template', + hidden: false, + namespaceType: 'single', + management: { + importableAndExportable: true, + getInAppUrl(obj: SavedObject) { + return { + path: `/app/integrations#/available/${obj.attributes.name}`, + uiCapabilitiesPath: 'advancedSettings.show', + }; + }, + getTitle(obj: SavedObject) { + return obj.attributes.displayName ?? obj.attributes.name; + }, + }, + mappings: { + dynamic: false, + properties: { + name: { + type: 'text', + }, + version: { + type: 'text', + }, + displayName: { + type: 'text', + }, + license: { + type: 'text', + }, + type: { + type: 'text', + }, + labels: { + type: 'text', + }, + author: { + type: 'text', + }, + description: { + type: 'text', + }, + sourceUrl: { + type: 'text', + }, + statics: { + type: 'nested', + }, + components: { + type: 'nested', + }, + assets: { + type: 'nested', + }, + sampleData: { + type: 'nested', + }, + }, + }, + }; + core.savedObjects.registerType(obsPanelType); core.savedObjects.registerType(integrationInstanceType); + core.savedObjects.registerType(integrationTemplateType); // Register server side APIs setupRoutes({ router, client: openSearchObservabilityClient, config }); diff --git a/server/routes/integrations/__tests__/integrations_router.test.ts b/server/routes/integrations/__tests__/integrations_router.test.ts index 15d2bac28b..5f6a7c39e5 100644 --- a/server/routes/integrations/__tests__/integrations_router.test.ts +++ b/server/routes/integrations/__tests__/integrations_router.test.ts @@ -26,12 +26,12 @@ describe('Data wrapper', () => { const result = await handleWithCallback( adaptorMock as IntegrationsAdaptor, responseMock as OpenSearchDashboardsResponseFactory, - callback + (callback as unknown) as (a: IntegrationsAdaptor) => Promise ); expect(callback).toHaveBeenCalled(); expect(responseMock.ok).toHaveBeenCalled(); - expect(result.body.data).toEqual({ test: 'data' }); + expect((result as { body?: unknown }).body).toEqual({ data: { test: 'data' } }); }); it('passes callback errors through', async () => { @@ -46,6 +46,6 @@ describe('Data wrapper', () => { expect(callback).toHaveBeenCalled(); expect(responseMock.custom).toHaveBeenCalled(); - expect(result.body).toEqual('test error'); + expect((result as { body?: unknown }).body).toEqual('test error'); }); }); diff --git a/server/routes/integrations/integrations_router.ts b/server/routes/integrations/integrations_router.ts index 46fe47768f..fba05b7b04 100644 --- a/server/routes/integrations/integrations_router.ts +++ b/server/routes/integrations/integrations_router.ts @@ -11,6 +11,7 @@ import { INTEGRATIONS_BASE } from '../../../common/constants/shared'; import { IntegrationsAdaptor } from '../../adaptors/integrations/integrations_adaptor'; import { OpenSearchDashboardsRequest, + OpenSearchDashboardsResponse, OpenSearchDashboardsResponseFactory, } from '../../../../../src/core/server/http/router'; import { IntegrationsManager } from '../../adaptors/integrations/integrations_manager'; @@ -29,19 +30,19 @@ import { IntegrationsManager } from '../../adaptors/integrations/integrations_ma * @callback callback A callback that will invoke a request on a provided adaptor. * @returns {Promise} An `OpenSearchDashboardsResponse` with the return data from the callback. */ -export const handleWithCallback = async ( +export const handleWithCallback = async ( adaptor: IntegrationsAdaptor, response: OpenSearchDashboardsResponseFactory, - callback: (a: IntegrationsAdaptor) => any -): Promise => { + callback: (a: IntegrationsAdaptor) => Promise +): Promise> => { try { const data = await callback(adaptor); return response.ok({ body: { data, }, - }); - } catch (err: any) { + }) as OpenSearchDashboardsResponse<{ data: T }>; + } catch (err) { console.error(`handleWithCallback: callback failed with error "${err.message}"`); return response.custom({ statusCode: err.statusCode || 500, @@ -63,7 +64,7 @@ export function registerIntegrationsRoute(router: IRouter) { path: `${INTEGRATIONS_BASE}/repository`, validate: false, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.getIntegrationTemplates() @@ -84,7 +85,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => { return a.loadIntegrationInstance( @@ -105,7 +106,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback( adaptor, @@ -130,7 +131,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); try { const requestPath = sanitize(request.params.path); @@ -141,7 +142,7 @@ export function registerIntegrationsRoute(router: IRouter) { }, body: result, }); - } catch (err: any) { + } catch (err) { return response.custom({ statusCode: err.statusCode ? err.statusCode : 500, body: err.message, @@ -159,7 +160,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.getSchemas(request.params.id) @@ -176,7 +177,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.getAssets(request.params.id) @@ -193,7 +194,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.getSampleData(request.params.id) @@ -206,7 +207,7 @@ export function registerIntegrationsRoute(router: IRouter) { path: `${INTEGRATIONS_BASE}/store`, validate: false, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.getIntegrationInstances({}) @@ -223,7 +224,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.deleteIntegrationInstance(request.params.id) @@ -240,7 +241,7 @@ export function registerIntegrationsRoute(router: IRouter) { }), }, }, - async (context, request, response): Promise => { + async (context, request, response): Promise => { const adaptor = getAdaptor(context, request); return handleWithCallback(adaptor, response, async (a: IntegrationsAdaptor) => a.getIntegrationInstance({